mi_check.c 153 KB
Newer Older
1
/*
2
   Copyright (c) 2000, 2011, Oracle and/or its affiliates
3

unknown's avatar
unknown committed
4 5
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
unknown's avatar
unknown committed
6
   the Free Software Foundation; version 2 of the License.
7

unknown's avatar
unknown committed
8 9 10 11
   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.
12

unknown's avatar
unknown committed
13 14
   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
15 16
   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
*/
unknown's avatar
unknown committed
17

18
/* Describe, check and repair of MyISAM tables */
unknown's avatar
unknown committed
19

20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
/*
  About checksum calculation.

  There are two types of checksums. Table checksum and row checksum.

  Row checksum is an additional byte at the end of dynamic length
  records. It must be calculated if the table is configured for them.
  Otherwise they must not be used. The variable
  MYISAM_SHARE::calc_checksum determines if row checksums are used.
  MI_INFO::checksum is used as temporary storage during row handling.
  For parallel repair we must assure that only one thread can use this
  variable. There is no problem on the write side as this is done by one
  thread only. But when checking a record after read this could go
  wrong. But since all threads read through a common read buffer, it is
  sufficient if only one thread checks it.

  Table checksum is an eight byte value in the header of the index file.
  It can be calculated even if row checksums are not used. The variable
  MI_CHECK::glob_crc is calculated over all records.
  MI_SORT_PARAM::calc_checksum determines if this should be done. This
  variable is not part of MI_CHECK because it must be set per thread for
  parallel repair. The global glob_crc must be changed by one thread
  only. And it is sufficient to calculate the checksum once only.
*/

45
#include "ftdefs.h"
unknown's avatar
unknown committed
46 47
#include <m_ctype.h>
#include <stdarg.h>
48
#include <my_getopt.h>
49
#ifdef HAVE_SYS_VADVISE_H
unknown's avatar
unknown committed
50 51 52 53 54
#include <sys/vadvise.h>
#endif
#ifdef HAVE_SYS_MMAN_H
#include <sys/mman.h>
#endif
55
#include "rt_index.h"
unknown's avatar
unknown committed
56 57 58 59 60 61 62 63

#ifndef USE_RAID
#define my_raid_create(A,B,C,D,E,F,G) my_create(A,B,C,G)
#define my_raid_delete(A,B,C) my_delete(A,B)
#endif

	/* Functions defined in this file */

64 65
static int check_k_link(HA_CHECK *param, MI_INFO *info,uint nr);
static int chk_index(HA_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
unknown's avatar
unknown committed
66 67 68 69
		     my_off_t page, uchar *buff, ha_rows *keys,
		     ha_checksum *key_checksum, uint level);
static uint isam_key_length(MI_INFO *info,MI_KEYDEF *keyinfo);
static ha_checksum calc_checksum(ha_rows count);
70
static int writekeys(MI_SORT_PARAM *sort_param);
71
static int sort_one_index(HA_CHECK *param, MI_INFO *info,MI_KEYDEF *keyinfo,
unknown's avatar
unknown committed
72
			  my_off_t pagepos, File new_file);
unknown's avatar
unknown committed
73 74 75 76
static int sort_key_read(MI_SORT_PARAM *sort_param,void *key);
static int sort_ft_key_read(MI_SORT_PARAM *sort_param,void *key);
static int sort_get_next_record(MI_SORT_PARAM *sort_param);
static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,const void *b);
77
static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a);
unknown's avatar
unknown committed
78
static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a);
unknown's avatar
unknown committed
79 80
static my_off_t get_record_for_key(MI_INFO *info,MI_KEYDEF *keyinfo,
				uchar *key);
81
static int sort_insert_key(MI_SORT_PARAM  *sort_param,
unknown's avatar
unknown committed
82
                           reg1 SORT_KEY_BLOCKS *key_block,
unknown's avatar
unknown committed
83
			   uchar *key, my_off_t prev_block);
unknown's avatar
unknown committed
84
static int sort_delete_record(MI_SORT_PARAM *sort_param);
85 86
/*static int flush_pending_blocks(HA_CHECK *param);*/
static SORT_KEY_BLOCKS	*alloc_key_blocks(HA_CHECK *param, uint blocks,
unknown's avatar
unknown committed
87
					  uint buffer_length);
88
static ha_checksum mi_byte_checksum(const uchar *buf, uint length);
89
static void set_data_file_type(MI_SORT_INFO *sort_info, MYISAM_SHARE *share);
unknown's avatar
unknown committed
90

91
void myisamchk_init(HA_CHECK *param)
unknown's avatar
unknown committed
92
{
93
  bzero((uchar*) param,sizeof(*param));
94
  /* Set all params that are not 0 */
unknown's avatar
unknown committed
95 96 97 98 99 100 101 102 103 104
  param->opt_follow_links=1;
  param->keys_in_use= ~(ulonglong) 0;
  param->search_after_block=HA_OFFSET_ERROR;
  param->use_buffers=USE_BUFFER_INIT;
  param->read_buffer_length=READ_BUFFER_INIT;
  param->write_buffer_length=READ_BUFFER_INIT;
  param->sort_buffer_length=SORT_BUFFER_INIT;
  param->sort_key_blocks=BUFFERS_WHEN_SORTING;
  param->tmpfile_createflag=O_RDWR | O_TRUNC | O_EXCL;
  param->myf_rw=MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL);
105
  param->max_record_length= LONGLONG_MAX;
106
  param->key_cache_block_size= KEY_CACHE_BLOCK_SIZE;
107
  param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL;
unknown's avatar
unknown committed
108 109
}

110 111
	/* Check the status flags for the table */

112
int chk_status(HA_CHECK *param, register MI_INFO *info)
113 114
{
  MYISAM_SHARE *share=info->s;
unknown's avatar
unknown committed
115

116 117 118 119 120 121
  if (mi_is_crashed_on_repair(info))
    mi_check_print_warning(param,
			   "Table is marked as crashed and last repair failed");
  else if (mi_is_crashed(info))
    mi_check_print_warning(param,
			   "Table is marked as crashed");
unknown's avatar
unknown committed
122
  if (share->state.open_count != (uint) (info->s->global_changed ? 1 : 0))
123
  {
unknown's avatar
unknown committed
124 125
    /* Don't count this as a real warning, as check can correct this ! */
    uint save=param->warning_printed;
126
    mi_check_print_warning(param,
127 128 129
			   share->state.open_count==1 ? 
			   "%d client is using or hasn't closed the table properly" : 
			   "%d clients are using or haven't closed the table properly",
130
			   share->state.open_count);
unknown's avatar
unknown committed
131 132 133
    /* If this will be fixed by the check, forget the warning */
    if (param->testflag & T_UPDATE_STATE)
      param->warning_printed=save;
134 135 136 137
  }
  return 0;
}

unknown's avatar
unknown committed
138 139
	/* Check delete links */

140
int chk_del(HA_CHECK *param, register MI_INFO *info, ulonglong test_flag)
unknown's avatar
unknown committed
141 142
{
  reg2 ha_rows i;
143
  uint delete_link_length;
144
  my_off_t empty,next_link,UNINIT_VAR(old_link);
unknown's avatar
unknown committed
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169
  char buff[22],buff2[22];
  DBUG_ENTER("chk_del");

  param->record_checksum=0;
  delete_link_length=((info->s->options & HA_OPTION_PACK_RECORD) ? 20 :
		      info->s->rec_reflength+1);

  if (!(test_flag & T_SILENT))
    puts("- check record delete-chain");

  next_link=info->s->state.dellink;
  if (info->state->del == 0)
  {
    if (test_flag & T_VERBOSE)
    {
      puts("No recordlinks");
    }
  }
  else
  {
    if (test_flag & T_VERBOSE)
      printf("Recordlinks:    ");
    empty=0;
    for (i= info->state->del ; i > 0L && next_link != HA_OFFSET_ERROR ; i--)
    {
170
      if (killed_ptr(param))
unknown's avatar
unknown committed
171
        DBUG_RETURN(1);
unknown's avatar
unknown committed
172 173 174 175
      if (test_flag & T_VERBOSE)
	printf(" %9s",llstr(next_link,buff));
      if (next_link >= info->state->data_file_length)
	goto wrong;
176
      if (my_pread(info->dfile, (uchar*) buff,delete_link_length,
unknown's avatar
unknown committed
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210
		   next_link,MYF(MY_NABP)))
      {
	if (test_flag & T_VERBOSE) puts("");
	mi_check_print_error(param,"Can't read delete-link at filepos: %s",
		    llstr(next_link,buff));
	DBUG_RETURN(1);
      }
      if (*buff != '\0')
      {
	if (test_flag & T_VERBOSE) puts("");
	mi_check_print_error(param,"Record at pos: %s is not remove-marked",
		    llstr(next_link,buff));
	goto wrong;
      }
      if (info->s->options & HA_OPTION_PACK_RECORD)
      {
	my_off_t prev_link=mi_sizekorr(buff+12);
	if (empty && prev_link != old_link)
	{
	  if (test_flag & T_VERBOSE) puts("");
	  mi_check_print_error(param,"Deleted block at %s doesn't point back at previous delete link",llstr(next_link,buff2));
	  goto wrong;
	}
	old_link=next_link;
	next_link=mi_sizekorr(buff+4);
	empty+=mi_uint3korr(buff+1);
      }
      else
      {
	param->record_checksum+=(ha_checksum) next_link;
	next_link=_mi_rec_pos(info->s,(uchar*) buff+1);
	empty+=info->s->base.pack_reclength;
      }
    }
211 212
    if (test_flag & T_VERBOSE)
      puts("\n");
unknown's avatar
unknown committed
213 214 215
    if (empty != info->state->empty)
    {
      mi_check_print_warning(param,
216 217 218
			     "Found %s deleted space in delete link chain. Should be %s",
			     llstr(empty,buff2),
			     llstr(info->state->empty,buff));
unknown's avatar
unknown committed
219
    }
220 221 222 223 224
    if (next_link != HA_OFFSET_ERROR)
    {
      mi_check_print_error(param,
			   "Found more than the expected %s deleted rows in delete link chain",
			   llstr(info->state->del, buff));
unknown's avatar
unknown committed
225
      goto wrong;
226 227 228 229 230 231 232 233 234
    }
    if (i != 0)
    {
      mi_check_print_error(param,
			   "Found %s deleted rows in delete link chain. Should be %s",
			   llstr(info->state->del - i, buff2),
			   llstr(info->state->del, buff));
      goto wrong;
    }
unknown's avatar
unknown committed
235 236
  }
  DBUG_RETURN(0);
237

unknown's avatar
unknown committed
238
wrong:
unknown's avatar
unknown committed
239
  param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
240 241 242 243 244 245 246 247
  if (test_flag & T_VERBOSE) puts("");
  mi_check_print_error(param,"record delete-link-chain corrupted");
  DBUG_RETURN(1);
} /* chk_del */


	/* Check delete links in index file */

248
static int check_k_link(HA_CHECK *param, register MI_INFO *info, uint nr)
unknown's avatar
unknown committed
249 250
{
  my_off_t next_link;
unknown's avatar
unknown committed
251
  uint block_size=(nr+1)*MI_MIN_KEY_BLOCK_LENGTH;
unknown's avatar
unknown committed
252
  ha_rows records;
253 254
  char llbuff[21], llbuff2[21];
  uchar *buff;
unknown's avatar
unknown committed
255
  DBUG_ENTER("check_k_link");
256
  DBUG_PRINT("enter", ("block_size: %u", block_size));
unknown's avatar
unknown committed
257 258

  if (param->testflag & T_VERBOSE)
259
    printf("block_size %4u:", block_size); /* purecov: tested */
unknown's avatar
unknown committed
260 261 262 263 264

  next_link=info->s->state.key_del[nr];
  records= (ha_rows) (info->state->key_file_length / block_size);
  while (next_link != HA_OFFSET_ERROR && records > 0)
  {
265
    if (killed_ptr(param))
unknown's avatar
unknown committed
266
      DBUG_RETURN(1);
unknown's avatar
unknown committed
267 268
    if (param->testflag & T_VERBOSE)
      printf("%16s",llstr(next_link,llbuff));
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288

    /* Key blocks must lay within the key file length entirely. */
    if (next_link + block_size > info->state->key_file_length)
    {
      /* purecov: begin tested */
      mi_check_print_error(param, "Invalid key block position: %s  "
                           "key block size: %u  file_length: %s",
                           llstr(next_link, llbuff), block_size,
                           llstr(info->state->key_file_length, llbuff2));
      DBUG_RETURN(1);
      /* purecov: end */
    }

    /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
    if (next_link & (MI_MIN_KEY_BLOCK_LENGTH - 1))
    {
      /* purecov: begin tested */
      mi_check_print_error(param, "Mis-aligned key block: %s  "
                           "minimum key block length: %u",
                           llstr(next_link, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
unknown's avatar
unknown committed
289
      DBUG_RETURN(1);
290 291 292 293 294 295 296 297
      /* purecov: end */
    }

    /*
      Read the key block with MI_MIN_KEY_BLOCK_LENGTH to find next link.
      If the key cache block size is smaller than block_size, we can so
      avoid unecessary eviction of cache block.
    */
unknown's avatar
unknown committed
298
    if (!(buff=key_cache_read(info->s->key_cache,
299
                              info->s->kfile, next_link, DFLT_INIT_HITS,
300
                              (uchar*) info->buff, MI_MIN_KEY_BLOCK_LENGTH,
301 302 303 304 305
                              MI_MIN_KEY_BLOCK_LENGTH, 1)))
    {
      /* purecov: begin tested */
      mi_check_print_error(param, "key cache read error for block: %s",
			   llstr(next_link,llbuff));
unknown's avatar
unknown committed
306
      DBUG_RETURN(1);
307 308
      /* purecov: end */
    }
unknown's avatar
unknown committed
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323
    next_link=mi_sizekorr(buff);
    records--;
    param->key_file_blocks+=block_size;
  }
  if (param->testflag & T_VERBOSE)
  {
    if (next_link != HA_OFFSET_ERROR)
      printf("%16s\n",llstr(next_link,llbuff));
    else
      puts("");
  }
  DBUG_RETURN (next_link != HA_OFFSET_ERROR);
} /* check_k_link */


unknown's avatar
unknown committed
324
	/* Check sizes of files */
unknown's avatar
unknown committed
325

326
int chk_size(HA_CHECK *param, register MI_INFO *info)
unknown's avatar
unknown committed
327 328 329 330 331 332 333 334
{
  int error=0;
  register my_off_t skr,size;
  char buff[22],buff2[22];
  DBUG_ENTER("chk_size");

  if (!(param->testflag & T_SILENT)) puts("- check file-size");

unknown's avatar
unknown committed
335 336 337
  /* The following is needed if called externally (not from myisamchk) */
  flush_key_blocks(info->s->key_cache,
		   info->s->kfile, FLUSH_FORCE_WRITE);
unknown's avatar
unknown committed
338

339
  size= my_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(MY_THREADSAFE));
unknown's avatar
unknown committed
340 341
  if ((skr=(my_off_t) info->state->key_file_length) != size)
  {
342
    /* Don't give error if file generated by myisampack */
343
    if (skr > size && mi_is_any_key_active(info->s->state.key_map))
unknown's avatar
unknown committed
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
    {
      error=1;
      mi_check_print_error(param,
			   "Size of indexfile is: %-8s        Should be: %s",
			   llstr(size,buff), llstr(skr,buff2));
    }
    else
      mi_check_print_warning(param,
			     "Size of indexfile is: %-8s      Should be: %s",
			     llstr(size,buff), llstr(skr,buff2));
  }
  if (!(param->testflag & T_VERY_SILENT) &&
      ! (info->s->options & HA_OPTION_COMPRESS_RECORD) &&
      ulonglong2double(info->state->key_file_length) >
      ulonglong2double(info->s->base.margin_key_file_length)*0.9)
    mi_check_print_warning(param,"Keyfile is almost full, %10s of %10s used",
			   llstr(info->state->key_file_length,buff),
			   llstr(info->s->base.max_key_file_length-1,buff));

  size=my_seek(info->dfile,0L,MY_SEEK_END,MYF(0));
  skr=(my_off_t) info->state->data_file_length;
  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
    skr+= MEMMAP_EXTRA_MARGIN;
#ifdef USE_RELOC
  if (info->data_file_type == STATIC_RECORD &&
      skr < (my_off_t) info->s->base.reloc*info->s->base.min_pack_length)
    skr=(my_off_t) info->s->base.reloc*info->s->base.min_pack_length;
#endif
  if (skr != size)
  {
374
    info->state->data_file_length=size;	/* Skip other errors */
unknown's avatar
unknown committed
375 376 377
    if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN)
    {
      error=1;
378
      mi_check_print_error(param,"Size of datafile is: %-9s         Should be: %s",
unknown's avatar
unknown committed
379
		    llstr(size,buff), llstr(skr,buff2));
unknown's avatar
unknown committed
380
      param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
381 382 383 384
    }
    else
    {
      mi_check_print_warning(param,
385
			     "Size of datafile is: %-9s       Should be: %s",
unknown's avatar
unknown committed
386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401
			     llstr(size,buff), llstr(skr,buff2));
    }
  }
  if (!(param->testflag & T_VERY_SILENT) &&
      !(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
      ulonglong2double(info->state->data_file_length) >
      (ulonglong2double(info->s->base.max_data_file_length)*0.9))
    mi_check_print_warning(param, "Datafile is almost full, %10s of %10s used",
			   llstr(info->state->data_file_length,buff),
			   llstr(info->s->base.max_data_file_length-1,buff2));
  DBUG_RETURN(error);
} /* chk_size */


	/* Check keys */

402
int chk_key(HA_CHECK *param, register MI_INFO *info)
unknown's avatar
unknown committed
403
{
404
  uint key,found_keys=0,full_text_keys=0,result=0;
unknown's avatar
unknown committed
405 406 407 408 409 410 411 412 413
  ha_rows keys;
  ha_checksum old_record_checksum,init_checksum;
  my_off_t all_keydata,all_totaldata,key_totlength,length;
  ulong   *rec_per_key_part;
  MYISAM_SHARE *share=info->s;
  MI_KEYDEF *keyinfo;
  char buff[22],buff2[22];
  DBUG_ENTER("chk_key");

414 415 416 417
  if (!(param->testflag & T_SILENT))
    puts("- check key delete-chain");

  param->key_file_blocks=info->s->base.keystart;
418
  for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
419 420 421 422 423 424 425
    if (check_k_link(param,info,key))
    {
      if (param->testflag & T_VERBOSE) puts("");
      mi_check_print_error(param,"key delete-link-chain corrupted");
      DBUG_RETURN(-1);
    }

unknown's avatar
unknown committed
426 427 428 429 430 431 432 433 434 435 436 437 438 439
  if (!(param->testflag & T_SILENT)) puts("- check index reference");

  all_keydata=all_totaldata=key_totlength=0;
  old_record_checksum=0;
  init_checksum=param->record_checksum;
  if (!(share->options &
	(HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
    old_record_checksum=calc_checksum(info->state->records+info->state->del-1)*
      share->base.pack_reclength;
  rec_per_key_part= param->rec_per_key_part;
  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
       rec_per_key_part+=keyinfo->keysegs, key++, keyinfo++)
  {
    param->key_crc[key]=0;
440
    if (! mi_is_key_active(share->state.key_map, key))
441 442 443
    {
      /* Remember old statistics for key */
      memcpy((char*) rec_per_key_part,
444 445
	     (char*) (share->state.rec_per_key_part +
		      (uint) (rec_per_key_part - param->rec_per_key_part)),
446
	     keyinfo->keysegs*sizeof(*rec_per_key_part));
unknown's avatar
unknown committed
447
      continue;
448
    }
unknown's avatar
unknown committed
449 450 451
    found_keys++;

    param->record_checksum=init_checksum;
452
    
unknown's avatar
unknown committed
453
    bzero((char*) &param->unique_count,sizeof(param->unique_count));
454 455
    bzero((char*) &param->notnull_count,sizeof(param->notnull_count));

unknown's avatar
unknown committed
456 457
    if ((!(param->testflag & T_SILENT)))
      printf ("- check data record references index: %d\n",key+1);
unknown's avatar
unknown committed
458
    if (keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL))
unknown's avatar
unknown committed
459
      full_text_keys++;
unknown's avatar
unknown committed
460
    if (share->state.key_root[key] == HA_OFFSET_ERROR &&
461
	(info->state->records == 0 || keyinfo->flag & HA_FULLTEXT))
462
      goto do_stat;
463 464
    if (!_mi_fetch_keypage(info,keyinfo,share->state.key_root[key],
                           DFLT_INIT_HITS,info->buff,0))
unknown's avatar
unknown committed
465 466 467
    {
      mi_check_print_error(param,"Can't read indexpage from filepos: %s",
		  llstr(share->state.key_root[key],buff));
468 469 470 471
      if (!(param->testflag & T_INFO))
	DBUG_RETURN(-1);
      result= -1;
      continue;
unknown's avatar
unknown committed
472 473 474 475 476 477 478 479 480
    }
    param->key_file_blocks+=keyinfo->block_length;
    keys=0;
    param->keydata=param->totaldata=0;
    param->key_blocks=0;
    param->max_level=0;
    if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
		  &keys, param->key_crc+key,1))
      DBUG_RETURN(-1);
481
    if(!(keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL)))
unknown's avatar
unknown committed
482 483 484 485 486
    {
      if (keys != info->state->records)
      {
	mi_check_print_error(param,"Found %s keys of %s",llstr(keys,buff),
		    llstr(info->state->records,buff2));
487
	if (!(param->testflag & T_INFO))
unknown's avatar
unknown committed
488
	DBUG_RETURN(-1);
489 490
	result= -1;
	continue;
unknown's avatar
unknown committed
491 492 493 494 495 496 497 498 499 500 501 502 503
      }
      if (found_keys - full_text_keys == 1 &&
	  ((share->options &
	    (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ||
	   (param->testflag & T_DONT_CHECK_CHECKSUM)))
	old_record_checksum=param->record_checksum;
      else if (old_record_checksum != param->record_checksum)
      {
	if (key)
	  mi_check_print_error(param,"Key %u doesn't point at same records that key 1",
		      key+1);
	else
	  mi_check_print_error(param,"Key 1 doesn't point at all records");
504 505 506 507
	if (!(param->testflag & T_INFO))
	  DBUG_RETURN(-1);
	result= -1;
	continue;
unknown's avatar
unknown committed
508 509 510 511
      }
    }
    if ((uint) share->base.auto_key -1 == key)
    {
unknown's avatar
unknown committed
512
      /* Check that auto_increment key is bigger than max key value */
513
      ulonglong auto_increment;
unknown's avatar
unknown committed
514 515
      info->lastinx=key;
      _mi_read_key_record(info, 0L, info->rec_buff);
516 517
      auto_increment= retrieve_auto_increment(info, info->rec_buff);
      if (auto_increment > info->s->state.auto_increment)
unknown's avatar
unknown committed
518
      {
519 520 521 522
        mi_check_print_warning(param, "Auto-increment value: %s is smaller "
                               "than max used value: %s",
                               llstr(info->s->state.auto_increment,buff2),
                               llstr(auto_increment, buff));
unknown's avatar
unknown committed
523 524 525
      }
      if (param->testflag & T_AUTO_INC)
      {
526 527 528 529
        set_if_bigger(info->s->state.auto_increment,
                      auto_increment);
        set_if_bigger(info->s->state.auto_increment,
                      param->auto_increment_value);
unknown's avatar
unknown committed
530
      }
unknown's avatar
unknown committed
531 532

      /* Check that there isn't a row with auto_increment = 0 in the table */
unknown's avatar
unknown committed
533
      mi_extra(info,HA_EXTRA_KEYREAD,0);
unknown's avatar
unknown committed
534
      bzero(info->lastkey,keyinfo->seg->length);
535
      if (!mi_rkey(info, info->rec_buff, key, (const uchar*) info->lastkey,
unknown's avatar
unknown committed
536
		   (key_part_map)1, HA_READ_KEY_EXACT))
unknown's avatar
unknown committed
537 538 539
      {
	/* Don't count this as a real warning, as myisamchk can't correct it */
	uint save=param->warning_printed;
540 541
        mi_check_print_warning(param, "Found row where the auto_increment "
                               "column has the value 0");
unknown's avatar
unknown committed
542 543
	param->warning_printed=save;
      }
unknown's avatar
unknown committed
544
      mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
unknown's avatar
unknown committed
545 546 547 548 549 550 551 552 553 554 555 556
    }

    length=(my_off_t) isam_key_length(info,keyinfo)*keys + param->key_blocks*2;
    if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L)
      printf("Key: %2d:  Keyblocks used: %3d%%  Packed: %4d%%  Max levels: %2d\n",
	     key+1,
	     (int) (my_off_t2double(param->keydata)*100.0/my_off_t2double(param->totaldata)),
	     (int) ((my_off_t2double(length) - my_off_t2double(param->keydata))*100.0/
		    my_off_t2double(length)),
	     param->max_level);
    all_keydata+=param->keydata; all_totaldata+=param->totaldata; key_totlength+=length;

557
do_stat:
unknown's avatar
unknown committed
558 559
    if (param->testflag & T_STATISTICS)
      update_key_parts(keyinfo, rec_per_key_part, param->unique_count,
560 561 562
                       param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
                       param->notnull_count: NULL, 
                       (ulonglong)info->state->records);
unknown's avatar
unknown committed
563 564 565 566 567 568 569 570 571 572
  }
  if (param->testflag & T_INFO)
  {
    if (all_totaldata != 0L && found_keys > 0)
      printf("Total:    Keyblocks used: %3d%%  Packed: %4d%%\n\n",
	     (int) (my_off_t2double(all_keydata)*100.0/
		    my_off_t2double(all_totaldata)),
	     (int) ((my_off_t2double(key_totlength) -
		     my_off_t2double(all_keydata))*100.0/
		     my_off_t2double(key_totlength)));
573
    else if (all_totaldata != 0L && mi_is_any_key_active(share->state.key_map))
unknown's avatar
unknown committed
574 575 576 577 578 579 580 581 582
      puts("");
  }
  if (param->key_file_blocks != info->state->key_file_length &&
      param->keys_in_use != ~(ulonglong) 0)
    mi_check_print_warning(param, "Some data are unreferenced in keyfile");
  if (found_keys != full_text_keys)
    param->record_checksum=old_record_checksum-init_checksum;	/* Remove delete links */
  else
    param->record_checksum=0;
unknown's avatar
unknown committed
583
  DBUG_RETURN(result);
unknown's avatar
unknown committed
584 585
} /* chk_key */

unknown's avatar
unknown committed
586

587
static int chk_index_down(HA_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
588 589 590 591
                     my_off_t page, uchar *buff, ha_rows *keys,
                     ha_checksum *key_checksum, uint level)
{
  char llbuff[22],llbuff2[22];
592
  DBUG_ENTER("chk_index_down");
593

594 595 596 597 598
  /* Key blocks must lay within the key file length entirely. */
  if (page + keyinfo->block_length > info->state->key_file_length)
  {
    /* purecov: begin tested */
    /* Give it a chance to fit in the real file size. */
599 600
    my_off_t max_length= my_seek(info->s->kfile, 0L, MY_SEEK_END,
                                 MYF(MY_THREADSAFE));
601 602 603 604 605
    mi_check_print_error(param, "Invalid key block position: %s  "
                         "key block size: %u  file_length: %s",
                         llstr(page, llbuff), keyinfo->block_length,
                         llstr(info->state->key_file_length, llbuff2));
    if (page + keyinfo->block_length > max_length)
606
      goto err;
607 608 609 610 611 612 613 614 615 616 617 618 619 620 621
    /* Fix the remebered key file length. */
    info->state->key_file_length= (max_length &
                                   ~ (my_off_t) (keyinfo->block_length - 1));
    /* purecov: end */
  }

  /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
  if (page & (MI_MIN_KEY_BLOCK_LENGTH - 1))
  {
    /* purecov: begin tested */
    mi_check_print_error(param, "Mis-aligned key block: %s  "
                         "minimum key block length: %u",
                         llstr(page, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
    goto err;
    /* purecov: end */
622
  }
623

624 625 626 627 628 629 630 631 632 633
  if (!_mi_fetch_keypage(info,keyinfo,page, DFLT_INIT_HITS,buff,0))
  {
    mi_check_print_error(param,"Can't read key from filepos: %s",
        llstr(page,llbuff));
    goto err;
  }
  param->key_file_blocks+=keyinfo->block_length;
  if (chk_index(param,info,keyinfo,page,buff,keys,key_checksum,level))
    goto err;

634 635 636
  DBUG_RETURN(0);

  /* purecov: begin tested */
637
err:
638 639
  DBUG_RETURN(1);
  /* purecov: end */
640
}
unknown's avatar
unknown committed
641

642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662

/*
  "Ignore NULLs" statistics collection method: process first index tuple.

  SYNOPSIS
    mi_collect_stats_nonulls_first()
      keyseg   IN     Array of key part descriptions
      notnull  INOUT  Array, notnull[i] = (number of {keypart1...keypart_i}
                                           tuples that don't contain NULLs)
      key      IN     Key values tuple

  DESCRIPTION
    Process the first index tuple - find out which prefix tuples don't
    contain NULLs, and update the array of notnull counters accordingly.
*/

static
void mi_collect_stats_nonulls_first(HA_KEYSEG *keyseg, ulonglong *notnull,
                                    uchar *key)
{
  uint first_null, kp;
663
  first_null= (uint) (ha_find_null(keyseg, key) - keyseg);
664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687
  /*
    All prefix tuples that don't include keypart_{first_null} are not-null
    tuples (and all others aren't), increment counters for them.
  */
  for (kp= 0; kp < first_null; kp++)
    notnull[kp]++;
}


/*
  "Ignore NULLs" statistics collection method: process next index tuple.

  SYNOPSIS
    mi_collect_stats_nonulls_next()
      keyseg   IN     Array of key part descriptions
      notnull  INOUT  Array, notnull[i] = (number of {keypart1...keypart_i}
                                           tuples that don't contain NULLs)
      prev_key IN     Previous key values tuple
      last_key IN     Next key values tuple

  DESCRIPTION
    Process the next index tuple:
    1. Find out which prefix tuples of last_key don't contain NULLs, and
       update the array of notnull counters accordingly.
688 689 690 691 692
    2. Find the first keypart number where the prev_key and last_key tuples
       are different(A), or last_key has NULL value(B), and return it, so the 
       caller can count number of unique tuples for each key prefix. We don't 
       need (B) to be counted, and that is compensated back in 
       update_key_parts().
693 694 695 696 697 698 699 700 701 702 703

  RETURN
    1 + number of first keypart where values differ or last_key tuple has NULL
*/

static
int mi_collect_stats_nonulls_next(HA_KEYSEG *keyseg, ulonglong *notnull,
                                  uchar *prev_key, uchar *last_key)
{
  uint diffs[2];
  uint first_null_seg, kp;
704
  HA_KEYSEG *seg;
705

706 707 708 709 710 711 712 713
  /* 
     Find the first keypart where values are different or either of them is
     NULL. We get results in diffs array:
     diffs[0]= 1 + number of first different keypart
     diffs[1]=offset: (last_key + diffs[1]) points to first value in
                      last_key that is NULL or different from corresponding
                      value in prev_key.
  */
714
  ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY, 
715 716
             SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs);
  seg= keyseg + diffs[0] - 1;
717

718
  /* Find first NULL in last_key */
719
  first_null_seg= (uint) (ha_find_null(seg, last_key + diffs[1]) - keyseg);
720 721 722 723 724 725 726 727 728 729 730 731
  for (kp= 0; kp < first_null_seg; kp++)
    notnull[kp]++;

  /* 
    Return 1+ number of first key part where values differ. Don't care if
    these were NULLs and not .... We compensate for that in
    update_key_parts.
  */
  return diffs[0];
}


unknown's avatar
unknown committed
732 733
	/* Check if index is ok */

734
static int chk_index(HA_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
unknown's avatar
unknown committed
735 736 737 738
		     my_off_t page, uchar *buff, ha_rows *keys,
		     ha_checksum *key_checksum, uint level)
{
  int flag;
739
  uint used_length,comp_flag,nod_flag,key_length=0;
740
  uchar key[HA_MAX_POSSIBLE_KEY_BUFF],*temp_buff,*keypos,*old_keypos,*endpos;
unknown's avatar
unknown committed
741
  my_off_t next_page,record;
unknown's avatar
unknown committed
742
  char llbuff[22];
743
  uint diff_pos[2];
unknown's avatar
unknown committed
744
  DBUG_ENTER("chk_index");
745
  DBUG_DUMP("buff",(uchar*) buff,mi_getint(buff));
unknown's avatar
unknown committed
746

747 748 749 750
  /* TODO: implement appropriate check for RTree keys */
  if (keyinfo->flag & HA_SPATIAL)
    DBUG_RETURN(0);

unknown's avatar
unknown committed
751 752
  if (!(temp_buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
  {
753
    mi_check_print_error(param,"Not enough memory for keyblock");
unknown's avatar
unknown committed
754 755 756 757
    DBUG_RETURN(-1);
  }

  if (keyinfo->flag & HA_NOSAME)
758 759 760 761
  {
    /* Not real duplicates */
    comp_flag= SEARCH_FIND | SEARCH_UPDATE | SEARCH_INSERT;
  }
unknown's avatar
unknown committed
762 763 764 765 766 767 768 769 770 771 772 773 774 775
  else
    comp_flag=SEARCH_SAME;			/* Keys in positionorder */
  nod_flag=mi_test_if_nod(buff);
  used_length=mi_getint(buff);
  keypos=buff+2+nod_flag;
  endpos=buff+used_length;

  param->keydata+=used_length; param->totaldata+=keyinfo->block_length;	/* INFO */
  param->key_blocks++;
  if (level > param->max_level)
    param->max_level=level;

  if (used_length > keyinfo->block_length)
  {
unknown's avatar
unknown committed
776 777
    mi_check_print_error(param,"Wrong pageinfo at page: %s",
			 llstr(page,llbuff));
unknown's avatar
unknown committed
778 779 780 781
    goto err;
  }
  for ( ;; )
  {
782
    if (killed_ptr(param))
unknown's avatar
unknown committed
783
      goto err;
784 785
    memcpy((char*) info->lastkey,(char*) key,key_length);
    info->lastkey_length=key_length;
unknown's avatar
unknown committed
786 787 788
    if (nod_flag)
    {
      next_page=_mi_kpos(nod_flag,keypos);
789 790
      if (chk_index_down(param,info,keyinfo,next_page,
                         temp_buff,keys,key_checksum,level+1))
unknown's avatar
unknown committed
791 792 793 794 795 796 797 798 799 800 801 802
	goto err;
    }
    old_keypos=keypos;
    if (keypos >= endpos ||
	(key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
      break;
    if (keypos > endpos)
    {
      mi_check_print_error(param,"Wrong key block length at page: %s",llstr(page,llbuff));
      goto err;
    }
    if ((*keys)++ &&
unknown's avatar
unknown committed
803
	(flag=ha_key_cmp(keyinfo->seg,info->lastkey,key,key_length,
804
			 comp_flag, diff_pos)) >=0)
unknown's avatar
unknown committed
805
    {
806 807 808
      DBUG_DUMP("old",info->lastkey, info->lastkey_length);
      DBUG_DUMP("new",key, key_length);
      DBUG_DUMP("new_in_page",old_keypos,(uint) (keypos-old_keypos));
unknown's avatar
unknown committed
809 810 811 812 813 814 815 816 817

      if (comp_flag & SEARCH_FIND && flag == 0)
	mi_check_print_error(param,"Found duplicated key at page %s",llstr(page,llbuff));
      else
	mi_check_print_error(param,"Key in wrong position at page %s",llstr(page,llbuff));
      goto err;
    }
    if (param->testflag & T_STATISTICS)
    {
818
      if (*keys != 1L)				/* not first_key */
unknown's avatar
unknown committed
819
      {
820 821 822
        if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
          ha_key_cmp(keyinfo->seg,info->lastkey,key,USE_WHOLE_KEY,
                     SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL,
823
                     diff_pos);
824 825
        else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
        {
826
          diff_pos[0]= mi_collect_stats_nonulls_next(keyinfo->seg, 
827 828 829
                                                  param->notnull_count,
                                                  info->lastkey, key);
        }
830
	param->unique_count[diff_pos[0]-1]++;
unknown's avatar
unknown committed
831
      }
832 833 834 835 836
      else
      {  
        if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
          mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count,
                                         key);
unknown's avatar
unknown committed
837 838
      }
    }
839
    (*key_checksum)+= mi_byte_checksum((uchar*) key,
unknown's avatar
unknown committed
840 841
				       key_length- info->s->rec_reflength);
    record= _mi_dpos(info,0,key+key_length);
842 843 844 845 846 847 848 849 850 851 852 853
    if (keyinfo->flag & HA_FULLTEXT) /* special handling for ft2 */
    {
      uint off;
      int  subkeys;
      get_key_full_length_rdonly(off, key);
      subkeys=ft_sintXkorr(key+off);
      if (subkeys < 0)
      {
        ha_rows tmp_keys=0;
        if (chk_index_down(param,info,&info->s->ft2_keyinfo,record,
                           temp_buff,&tmp_keys,key_checksum,1))
          goto err;
854 855
        if (tmp_keys + subkeys)
        {
unknown's avatar
unknown committed
856 857 858 859 860
          mi_check_print_error(param,
                               "Number of words in the 2nd level tree "
                               "does not match the number in the header. "
                               "Parent word in on the page %s, offset %u",
                               llstr(page,llbuff), (uint) (old_keypos-buff));
861 862
          goto err;
        }
863 864 865 866 867
        (*keys)+=tmp_keys-1;
        continue;
      }
      /* fall through */
    }
unknown's avatar
unknown committed
868 869 870
    if (record >= info->state->data_file_length)
    {
#ifndef DBUG_OFF
unknown's avatar
unknown committed
871
      char llbuff2[22], llbuff3[22];
unknown's avatar
unknown committed
872 873 874 875 876
#endif
      mi_check_print_error(param,"Found key at page %s that points to record outside datafile",llstr(page,llbuff));
      DBUG_PRINT("test",("page: %s  record: %s  filelength: %s",
			 llstr(page,llbuff),llstr(record,llbuff2),
			 llstr(info->state->data_file_length,llbuff3)));
877 878
      DBUG_DUMP("key",key,key_length);
      DBUG_DUMP("new_in_page",old_keypos,(uint) (keypos-old_keypos));
unknown's avatar
unknown committed
879 880 881 882 883 884 885 886 887 888
      goto err;
    }
    param->record_checksum+=(ha_checksum) record;
  }
  if (keypos != endpos)
  {
    mi_check_print_error(param,"Keyblock size at page %s is not correct.  Block length: %d  key length: %d",
                llstr(page,llbuff), used_length, (keypos - buff));
    goto err;
  }
889
  my_afree((uchar*) temp_buff);
unknown's avatar
unknown committed
890 891
  DBUG_RETURN(0);
 err:
892
  my_afree((uchar*) temp_buff);
unknown's avatar
unknown committed
893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925
  DBUG_RETURN(1);
} /* chk_index */


	/* Calculate a checksum of 1+2+3+4...N = N*(N+1)/2 without overflow */

static ha_checksum calc_checksum(ha_rows count)
{
  ulonglong sum,a,b;
  DBUG_ENTER("calc_checksum");

  sum=0;
  a=count; b=count+1;
  if (a & 1)
    b>>=1;
  else
    a>>=1;
  while (b)
  {
    if (b & 1)
      sum+=a;
    a<<=1; b>>=1;
  }
  DBUG_PRINT("exit",("sum: %lx",(ulong) sum));
  DBUG_RETURN((ha_checksum) sum);
} /* calc_checksum */


	/* Calc length of key in normal isam */

static uint isam_key_length(MI_INFO *info, register MI_KEYDEF *keyinfo)
{
  uint length;
unknown's avatar
unknown committed
926
  HA_KEYSEG *keyseg;
unknown's avatar
unknown committed
927 928 929 930 931 932 933 934 935 936 937 938 939
  DBUG_ENTER("isam_key_length");

  length= info->s->rec_reflength;
  for (keyseg=keyinfo->seg ; keyseg->type ; keyseg++)
    length+= keyseg->length;

  DBUG_PRINT("exit",("length: %d",length));
  DBUG_RETURN(length);
} /* key_length */


	/* Check that record-link is ok */

940
int chk_data_link(HA_CHECK *param, MI_INFO *info, my_bool extend)
unknown's avatar
unknown committed
941 942
{
  int	error,got_error,flag;
943
  uint	key, UNINIT_VAR(left_length), b_type;
unknown's avatar
unknown committed
944
  ha_rows records,del_blocks;
945
  my_off_t used,empty,pos,splits,UNINIT_VAR(start_recpos),
unknown's avatar
unknown committed
946
	   del_length,link_used,start_block;
Staale Smedseng's avatar
Staale Smedseng committed
947
  uchar	*record= 0, *UNINIT_VAR(to);
unknown's avatar
unknown committed
948 949
  char llbuff[22],llbuff2[22],llbuff3[22];
  ha_checksum intern_record_checksum;
950
  ha_checksum key_checksum[HA_MAX_POSSIBLE_KEY];
unknown's avatar
unknown committed
951 952 953 954 955 956 957 958 959 960 961 962
  MI_KEYDEF *keyinfo;
  MI_BLOCK_INFO block_info;
  DBUG_ENTER("chk_data_link");

  if (!(param->testflag & T_SILENT))
  {
    if (extend)
      puts("- check records and index references");
    else
      puts("- check record links");
  }

963
  if (!mi_alloc_rec_buff(info, -1, &record))
unknown's avatar
unknown committed
964
  {
965
    mi_check_print_error(param,"Not enough memory for record");
unknown's avatar
unknown committed
966 967 968 969 970 971
    DBUG_RETURN(-1);
  }
  records=del_blocks=0;
  used=link_used=splits=del_length=0;
  intern_record_checksum=param->glob_crc=0;
  got_error=error=0;
972
  empty=info->s->pack.header_length;
unknown's avatar
unknown committed
973

974
  pos=my_b_tell(&param->read_cache);
unknown's avatar
unknown committed
975 976 977
  bzero((char*) key_checksum, info->s->base.keys * sizeof(key_checksum[0]));
  while (pos < info->state->data_file_length)
  {
978
    if (killed_ptr(param))
unknown's avatar
unknown committed
979
      goto err2;
unknown's avatar
unknown committed
980
    switch (info->s->data_file_type) {
unknown's avatar
unknown committed
981 982 983
    case BLOCK_RECORD:
      DBUG_ASSERT(0);                           /* Impossible */
      break;
unknown's avatar
unknown committed
984
    case STATIC_RECORD:
985
      if (my_b_read(&param->read_cache,(uchar*) record,
unknown's avatar
unknown committed
986 987 988 989 990 991 992 993 994 995 996
		    info->s->base.pack_reclength))
	goto err;
      start_recpos=pos;
      pos+=info->s->base.pack_reclength;
      splits++;
      if (*record == '\0')
      {
	del_blocks++;
	del_length+=info->s->base.pack_reclength;
	continue;					/* Record removed */
      }
997
      param->glob_crc+= (*info->s->calc_check_checksum)(info,record);
unknown's avatar
unknown committed
998 999 1000 1001 1002 1003 1004
      used+=info->s->base.pack_reclength;
      break;
    case DYNAMIC_RECORD:
      flag=block_info.second_read=0;
      block_info.next_filepos=pos;
      do
      {
1005
	if (_mi_read_cache(&param->read_cache,(uchar*) block_info.header,
1006 1007 1008
			   (start_block=block_info.next_filepos),
			   sizeof(block_info.header),
			   (flag ? 0 : READING_NEXT) | READING_HEADER))
unknown's avatar
unknown committed
1009 1010 1011
	  goto err;
	if (start_block & (MI_DYN_ALIGN_SIZE-1))
	{
1012 1013
	  mi_check_print_error(param,"Wrong aligned block at %s",
			       llstr(start_block,llbuff));
unknown's avatar
unknown committed
1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035
	  goto err2;
	}
	b_type=_mi_get_block_info(&block_info,-1,start_block);
	if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
		      BLOCK_FATAL_ERROR))
	{
	  if (b_type & BLOCK_SYNC_ERROR)
	  {
	    if (flag)
	    {
	      mi_check_print_error(param,"Unexpected byte: %d at link: %s",
			  (int) block_info.header[0],
			  llstr(start_block,llbuff));
	      goto err2;
	    }
	    pos=block_info.filepos+block_info.block_len;
	    goto next;
	  }
	  if (b_type & BLOCK_DELETED)
	  {
	    if (block_info.block_len < info->s->base.min_block_length)
	    {
1036 1037 1038
	      mi_check_print_error(param,
				   "Deleted block with impossible length %lu at %s",
				   block_info.block_len,llstr(pos,llbuff));
unknown's avatar
unknown committed
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056
	      goto err2;
	    }
	    if ((block_info.next_filepos != HA_OFFSET_ERROR &&
		 block_info.next_filepos >= info->state->data_file_length) ||
		(block_info.prev_filepos != HA_OFFSET_ERROR &&
		 block_info.prev_filepos >= info->state->data_file_length))
	    {
	      mi_check_print_error(param,"Delete link points outside datafile at %s",
			  llstr(pos,llbuff));
	      goto err2;
	    }
	    del_blocks++;
	    del_length+=block_info.block_len;
	    pos=block_info.filepos+block_info.block_len;
	    splits++;
	    goto next;
	  }
	  mi_check_print_error(param,"Wrong bytesec: %d-%d-%d at linkstart: %s",
1057 1058 1059
			       block_info.header[0],block_info.header[1],
			       block_info.header[2],
			       llstr(start_block,llbuff));
unknown's avatar
unknown committed
1060 1061 1062 1063 1064
	  goto err2;
	}
	if (info->state->data_file_length < block_info.filepos+
	    block_info.block_len)
	{
1065 1066 1067
	  mi_check_print_error(param,
			       "Recordlink that points outside datafile at %s",
			       llstr(pos,llbuff));
unknown's avatar
unknown committed
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
	  got_error=1;
	  break;
	}
	splits++;
	if (!flag++)				/* First block */
	{
	  start_recpos=pos;
	  pos=block_info.filepos+block_info.block_len;
	  if (block_info.rec_len > (uint) info->s->base.max_pack_length)
	  {
1078 1079 1080
	    mi_check_print_error(param,"Found too long record (%lu) at %s",
				 (ulong) block_info.rec_len,
				 llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1081 1082 1083 1084 1085
	    got_error=1;
	    break;
	  }
	  if (info->s->base.blobs)
	  {
1086
	    if (!(to= mi_alloc_rec_buff(info, block_info.rec_len,
1087
					&info->rec_buff)))
unknown's avatar
unknown committed
1088
	    {
1089 1090 1091 1092
	      mi_check_print_error(param,
				   "Not enough memory (%lu) for blob at %s",
				   (ulong) block_info.rec_len,
				   llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1093 1094 1095 1096 1097 1098 1099 1100 1101 1102
	      got_error=1;
	      break;
	    }
	  }
	  else
	    to= info->rec_buff;
	  left_length=block_info.rec_len;
	}
	if (left_length < block_info.data_len)
	{
1103 1104 1105 1106 1107
	  mi_check_print_error(param,"Found too long record (%lu) at %s",
			       (ulong) block_info.data_len,
			       llstr(start_recpos,llbuff));
	  got_error=1;
	  break;
unknown's avatar
unknown committed
1108
	}
1109
	if (_mi_read_cache(&param->read_cache,(uchar*) to,block_info.filepos,
1110 1111
			   (uint) block_info.data_len,
			   flag == 1 ? READING_NEXT : 0))
unknown's avatar
unknown committed
1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
	  goto err;
	to+=block_info.data_len;
	link_used+= block_info.filepos-start_block;
	used+= block_info.filepos - start_block + block_info.data_len;
	empty+=block_info.block_len-block_info.data_len;
	left_length-=block_info.data_len;
	if (left_length)
	{
	  if (b_type & BLOCK_LAST)
	  {
1122 1123 1124 1125 1126
	    mi_check_print_error(param,
				 "Wrong record length %s of %s at %s",
				 llstr(block_info.rec_len-left_length,llbuff),
				 llstr(block_info.rec_len, llbuff2),
				 llstr(start_recpos,llbuff3));
unknown's avatar
unknown committed
1127 1128 1129 1130 1131
	    got_error=1;
	    break;
	  }
	  if (info->state->data_file_length < block_info.next_filepos)
	  {
1132 1133 1134
	    mi_check_print_error(param,
				 "Found next-recordlink that points outside datafile at %s",
				 llstr(block_info.filepos,llbuff));
unknown's avatar
unknown committed
1135 1136 1137 1138 1139 1140 1141 1142 1143 1144
	    got_error=1;
	    break;
	  }
	}
      } while (left_length);
      if (! got_error)
      {
	if (_mi_rec_unpack(info,record,info->rec_buff,block_info.rec_len) ==
	    MY_FILE_ERROR)
	{
1145 1146
	  mi_check_print_error(param,"Found wrong record at %s",
			       llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1147 1148 1149 1150
	  got_error=1;
	}
	else
	{
1151
	  info->checksum= (*info->s->calc_check_checksum)(info,record);
unknown's avatar
unknown committed
1152 1153
	  if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE))
	  {
1154 1155
	    if (_mi_rec_check(info,record, info->rec_buff,block_info.rec_len,
                              test(info->s->calc_checksum)))
unknown's avatar
unknown committed
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169
	    {
	      mi_check_print_error(param,"Found wrong packed record at %s",
			  llstr(start_recpos,llbuff));
	      got_error=1;
	    }
	  }
	  if (!got_error)
	    param->glob_crc+= info->checksum;
	}
      }
      else if (!flag)
	pos=block_info.filepos+block_info.block_len;
      break;
    case COMPRESSED_RECORD:
1170
      if (_mi_read_cache(&param->read_cache,(uchar*) block_info.header, pos,
1171
			 info->s->pack.ref_length, READING_NEXT))
unknown's avatar
unknown committed
1172 1173 1174
	goto err;
      start_recpos=pos;
      splits++;
1175 1176
      VOID(_mi_pack_get_block_info(info, &info->bit_buff, &block_info,
                                   &info->rec_buff, -1, start_recpos));
unknown's avatar
unknown committed
1177 1178 1179 1180
      pos=block_info.filepos+block_info.rec_len;
      if (block_info.rec_len < (uint) info->s->min_pack_length ||
	  block_info.rec_len > (uint) info->s->max_pack_length)
      {
unknown's avatar
unknown committed
1181 1182 1183
	mi_check_print_error(param,
			     "Found block with wrong recordlength: %d at %s",
			     block_info.rec_len, llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1184 1185 1186
	got_error=1;
	break;
      }
1187
      if (_mi_read_cache(&param->read_cache,(uchar*) info->rec_buff,
1188
			block_info.filepos, block_info.rec_len, READING_NEXT))
unknown's avatar
unknown committed
1189
	goto err;
1190 1191
      if (_mi_pack_rec_unpack(info, &info->bit_buff, record,
                              info->rec_buff, block_info.rec_len))
unknown's avatar
unknown committed
1192
      {
unknown's avatar
unknown committed
1193 1194
	mi_check_print_error(param,"Found wrong record at %s",
			     llstr(start_recpos,llbuff));
unknown's avatar
unknown committed
1195 1196
	got_error=1;
      }
1197
      param->glob_crc+= (*info->s->calc_check_checksum)(info,record);
unknown's avatar
unknown committed
1198 1199
      link_used+= (block_info.filepos - start_recpos);
      used+= (pos-start_recpos);
1200
      break;
unknown's avatar
unknown committed
1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215
    } /* switch */
    if (! got_error)
    {
      intern_record_checksum+=(ha_checksum) start_recpos;
      records++;
      if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0)
      {
	printf("%s\r", llstr(records,llbuff)); VOID(fflush(stdout));
      }

      /* Check if keys match the record */

      for (key=0,keyinfo= info->s->keyinfo; key < info->s->base.keys;
	   key++,keyinfo++)
      {
1216
        if (mi_is_key_active(info->s->state.key_map, key))
unknown's avatar
unknown committed
1217
	{
1218
          if(!(keyinfo->flag & HA_FULLTEXT))
unknown's avatar
unknown committed
1219 1220 1221 1222 1223 1224 1225 1226
	  {
	    uint key_length=_mi_make_key(info,key,info->lastkey,record,
					 start_recpos);
	    if (extend)
	    {
	      /* We don't need to lock the key tree here as we don't allow
		 concurrent threads when running myisamchk
	      */
1227 1228 1229
              int search_result=
#ifdef HAVE_RTREE_KEYS
                (keyinfo->flag & HA_SPATIAL) ?
1230
                rtree_find_first(info, key, info->lastkey, key_length,
1231
                                 MBR_EQUAL | MBR_DATA) : 
1232
#endif
1233 1234 1235 1236
                _mi_search(info,keyinfo,info->lastkey,key_length,
                           SEARCH_SAME, info->s->state.key_root[key]);
              if (search_result)
              {
1237 1238
                mi_check_print_error(param,"Record at: %10s  "
                                     "Can't find key for index: %2d",
1239 1240 1241 1242
                                     llstr(start_recpos,llbuff),key+1);
                if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
                  goto err2;
              }
unknown's avatar
unknown committed
1243 1244
	    }
	    else
1245
	      key_checksum[key]+=mi_byte_checksum((uchar*) info->lastkey,
unknown's avatar
unknown committed
1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275
						  key_length);
	  }
	}
      }
    }
    else
    {
      got_error=0;
      if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
	goto err2;
    }
  next:;				/* Next record */
  }
  if (param->testflag & T_WRITE_LOOP)
  {
    VOID(fputs("          \r",stdout)); VOID(fflush(stdout));
  }
  if (records != info->state->records)
  {
    mi_check_print_error(param,"Record-count is not ok; is %-10s   Should be: %s",
		llstr(records,llbuff), llstr(info->state->records,llbuff2));
    error=1;
  }
  else if (param->record_checksum &&
	   param->record_checksum != intern_record_checksum)
  {
    mi_check_print_error(param,
			 "Keypointers and record positions doesn't match");
    error=1;
  }
1276
  else if (param->glob_crc != info->state->checksum &&
unknown's avatar
unknown committed
1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
	   (info->s->options &
	    (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD)))
  {
    mi_check_print_warning(param,
			   "Record checksum is not the same as checksum stored in the index file\n");
    error=1;
  }
  else if (!extend)
  {
    for (key=0 ; key < info->s->base.keys;  key++)
    {
      if (key_checksum[key] != param->key_crc[key] &&
1289
          !(info->s->keyinfo[key].flag & (HA_FULLTEXT | HA_SPATIAL)))
unknown's avatar
unknown committed
1290 1291 1292 1293 1294 1295 1296 1297
      {
	mi_check_print_error(param,"Checksum for key: %2d doesn't match checksum for records",
		    key+1);
	error=1;
      }
    }
  }

1298 1299 1300 1301 1302 1303 1304
  if (del_length != info->state->empty)
  {
    mi_check_print_warning(param,
			   "Found %s deleted space.   Should be %s",
			   llstr(del_length,llbuff2),
			   llstr(info->state->empty,llbuff));
  }
unknown's avatar
unknown committed
1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325
  if (used+empty+del_length != info->state->data_file_length)
  {
    mi_check_print_warning(param,
			   "Found %s record-data and %s unused data and %s deleted-data",
			   llstr(used,llbuff),llstr(empty,llbuff2),
			   llstr(del_length,llbuff3));
    mi_check_print_warning(param,
			   "Total %s, Should be: %s",
			   llstr((used+empty+del_length),llbuff),
			   llstr(info->state->data_file_length,llbuff2));
  }
  if (del_blocks != info->state->del)
  {
    mi_check_print_warning(param,
			   "Found %10s deleted blocks       Should be: %s",
			   llstr(del_blocks,llbuff),
			   llstr(info->state->del,llbuff2));
  }
  if (splits != info->s->state.split)
  {
    mi_check_print_warning(param,
1326
			   "Found %10s parts. Should be: %s",
unknown's avatar
unknown committed
1327 1328 1329 1330 1331 1332 1333 1334 1335 1336
			   llstr(splits,llbuff),
			   llstr(info->s->state.split,llbuff2));
  }
  if (param->testflag & T_INFO)
  {
    if (param->warning_printed || param->error_printed)
      puts("");
    if (used != 0 && ! param->error_printed)
    {
      printf("Records:%18s    M.recordlength:%9lu   Packed:%14.0f%%\n",
unknown's avatar
unknown committed
1337
	     llstr(records,llbuff), (long)((used-link_used)/records),
unknown's avatar
unknown committed
1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354
	     (info->s->base.blobs ? 0.0 :
	      (ulonglong2double((ulonglong) info->s->base.reclength*records)-
	       my_off_t2double(used))/
	      ulonglong2double((ulonglong) info->s->base.reclength*records)*100.0));
      printf("Recordspace used:%9.0f%%   Empty space:%12d%%  Blocks/Record: %6.2f\n",
	     (ulonglong2double(used-link_used)/ulonglong2double(used-link_used+empty)*100.0),
	     (!records ? 100 : (int) (ulonglong2double(del_length+empty)/
				      my_off_t2double(used)*100.0)),
	     ulonglong2double(splits - del_blocks) / records);
    }
    printf("Record blocks:%12s    Delete blocks:%10s\n",
	   llstr(splits-del_blocks,llbuff),llstr(del_blocks,llbuff2));
    printf("Record data:  %12s    Deleted data: %10s\n",
	   llstr(used-link_used,llbuff),llstr(del_length,llbuff2));
    printf("Lost space:   %12s    Linkdata:     %10s\n",
	   llstr(empty,llbuff),llstr(link_used,llbuff2));
  }
1355
  my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
unknown's avatar
unknown committed
1356 1357
  DBUG_RETURN (error);
 err:
unknown's avatar
unknown committed
1358
  mi_check_print_error(param,"got error: %d when reading datafile at record: %s",my_errno, llstr(records,llbuff));
unknown's avatar
unknown committed
1359
 err2:
1360
  my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
unknown's avatar
unknown committed
1361
  param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
1362 1363 1364 1365
  DBUG_RETURN(1);
} /* chk_data_link */


1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418
/**
  @brief Drop all indexes

  @param[in]    param           check parameters
  @param[in]    info            MI_INFO handle
  @param[in]    force           if to force drop all indexes

  @return       status
    @retval     0               OK
    @retval     != 0            Error

  @note
    Once allocated, index blocks remain part of the key file forever.
    When indexes are disabled, no block is freed. When enabling indexes,
    no block is freed either. The new indexes are create from new
    blocks. (Bug #4692)

    Before recreating formerly disabled indexes, the unused blocks
    must be freed. There are two options to do this:
    - Follow the tree of disabled indexes, add all blocks to the
      deleted blocks chain. Would require a lot of random I/O.
    - Drop all blocks by clearing all index root pointers and all
      delete chain pointers and resetting key_file_length to the end
      of the index file header. This requires to recreate all indexes,
      even those that may still be intact.
    The second method is probably faster in most cases.

    When disabling indexes, MySQL disables either all indexes or all
    non-unique indexes. When MySQL [re-]enables disabled indexes
    (T_CREATE_MISSING_KEYS), then we either have "lost" blocks in the
    index file, or there are no non-unique indexes. In the latter case,
    mi_repair*() would not be called as there would be no disabled
    indexes.

    If there would be more unique indexes than disabled (non-unique)
    indexes, we could do the first method. But this is not implemented
    yet. By now we drop and recreate all indexes when repair is called.

    However, there is an exception. Sometimes MySQL disables non-unique
    indexes when the table is empty (e.g. when copying a table in
    mysql_alter_table()). When enabling the non-unique indexes, they
    are still empty. So there is no index block that can be lost. This
    optimization is implemented in this function.

    Note that in normal repair (T_CREATE_MISSING_KEYS not set) we
    recreate all enabled indexes unconditonally. We do not change the
    key_map. Otherwise we invert the key map temporarily (outside of
    this function) and recreate the then "seemingly" enabled indexes.
    When we cannot use the optimization, and drop all indexes, we
    pretend that all indexes were disabled. By the inversion, we will
    then recrate all indexes.
*/

unknown's avatar
unknown committed
1419
static int mi_drop_all_indexes(HA_CHECK *param, MI_INFO *info, my_bool force)
1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477
{
  MYISAM_SHARE *share= info->s;
  MI_STATE_INFO *state= &share->state;
  uint i;
  int error;
  DBUG_ENTER("mi_drop_all_indexes");

  /*
    If any of the disabled indexes has a key block assigned, we must
    drop and recreate all indexes to avoid losing index blocks.

    If we want to recreate disabled indexes only _and_ all of these
    indexes are empty, we don't need to recreate the existing indexes.
  */
  if (!force && (param->testflag & T_CREATE_MISSING_KEYS))
  {
    DBUG_PRINT("repair", ("creating missing indexes"));
    for (i= 0; i < share->base.keys; i++)
    {
      DBUG_PRINT("repair", ("index #: %u  key_root: 0x%lx  active: %d",
                            i, (long) state->key_root[i],
                            mi_is_key_active(state->key_map, i)));
      if ((state->key_root[i] != HA_OFFSET_ERROR) &&
          !mi_is_key_active(state->key_map, i))
      {
        /*
          This index has at least one key block and it is disabled.
          We would lose its block(s) if would just recreate it.
          So we need to drop and recreate all indexes.
        */
        DBUG_PRINT("repair", ("nonempty and disabled: recreate all"));
        break;
      }
    }
    if (i >= share->base.keys)
    {
      /*
        All of the disabled indexes are empty. We can just recreate them.
        Flush dirty blocks of this index file from key cache and remove
        all blocks of this index file from key cache.
      */
      DBUG_PRINT("repair", ("all disabled are empty: create missing"));
      error= flush_key_blocks(share->key_cache, share->kfile,
                              FLUSH_FORCE_WRITE);
      goto end;
    }
    /*
      We do now drop all indexes and declare them disabled. With the
      T_CREATE_MISSING_KEYS flag, mi_repair*() will recreate all
      disabled indexes and enable them.
    */
    mi_clear_all_keys_active(state->key_map);
    DBUG_PRINT("repair", ("declared all indexes disabled"));
  }

  /* Remove all key blocks of this index file from key cache. */
  if ((error= flush_key_blocks(share->key_cache, share->kfile,
                               FLUSH_IGNORE_CHANGED)))
1478
    goto end; /* purecov: inspected */
1479 1480 1481 1482 1483 1484

  /* Clear index root block pointers. */
  for (i= 0; i < share->base.keys; i++)
    state->key_root[i]= HA_OFFSET_ERROR;

  /* Clear the delete chains. */
1485
  for (i= 0; i < state->header.max_block_size_index; i++)
1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498
    state->key_del[i]= HA_OFFSET_ERROR;

  /* Reset index file length to end of index file header. */
  info->state->key_file_length= share->base.keystart;

  DBUG_PRINT("repair", ("dropped all indexes"));
  /* error= 0; set by last (error= flush_key_bocks()). */

 end:
  DBUG_RETURN(error);
}


unknown's avatar
unknown committed
1499 1500 1501
	/* Recover old table by reading each record and writing all keys */
	/* Save new datafile-name in temp_filename */

1502
int mi_repair(HA_CHECK *param, register MI_INFO *info,
1503
	      char * name, int rep_quick)
unknown's avatar
unknown committed
1504 1505 1506 1507 1508 1509 1510
{
  int error,got_error;
  ha_rows start_records,new_header_length;
  my_off_t del;
  File new_file;
  MYISAM_SHARE *share=info->s;
  char llbuff[22],llbuff2[22];
1511
  MI_SORT_INFO sort_info;
unknown's avatar
unknown committed
1512
  MI_SORT_PARAM sort_param;
unknown's avatar
unknown committed
1513 1514
  DBUG_ENTER("mi_repair");

unknown's avatar
unknown committed
1515 1516
  bzero((char *)&sort_info, sizeof(sort_info));
  bzero((char *)&sort_param, sizeof(sort_param));
unknown's avatar
unknown committed
1517
  start_records=info->state->records;
1518 1519
  new_header_length=(param->testflag & T_UNPACK) ? 0L :
    share->pack.header_length;
unknown's avatar
unknown committed
1520 1521
  got_error=1;
  new_file= -1;
unknown's avatar
unknown committed
1522
  sort_param.sort_info=&sort_info;
1523

unknown's avatar
unknown committed
1524 1525
  if (!(param->testflag & T_SILENT))
  {
1526
    printf("- recovering (with keycache) MyISAM-table '%s'\n",name);
unknown's avatar
unknown committed
1527 1528
    printf("Data records: %s\n", llstr(info->state->records,llbuff));
  }
1529
  param->testflag|=T_REP; /* for easy checking */
unknown's avatar
unknown committed
1530

1531 1532 1533
  if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
    param->testflag|=T_CALC_CHECKSUM;

1534 1535
  DBUG_ASSERT(param->use_buffers < SIZE_T_MAX);

unknown's avatar
unknown committed
1536
  if (!param->using_global_keycache)
unknown's avatar
unknown committed
1537
    VOID(init_key_cache(dflt_key_cache, param->key_cache_block_size,
1538
                        (size_t) param->use_buffers, 0, 0));
unknown's avatar
unknown committed
1539 1540 1541 1542

  if (init_io_cache(&param->read_cache,info->dfile,
		    (uint) param->read_buffer_length,
		    READ_CACHE,share->pack.header_length,1,MYF(MY_WME)))
1543 1544
  {
    bzero(&info->rec_cache,sizeof(info->rec_cache));
unknown's avatar
unknown committed
1545
    goto err;
1546
  }
unknown's avatar
unknown committed
1547 1548 1549 1550 1551 1552
  if (!rep_quick)
    if (init_io_cache(&info->rec_cache,-1,(uint) param->write_buffer_length,
		      WRITE_CACHE, new_header_length, 1,
		      MYF(MY_WME | MY_WAIT_IF_FULL)))
      goto err;
  info->opt_flag|=WRITE_CACHE_USED;
1553
  if (!mi_alloc_rec_buff(info, -1, &sort_param.record) ||
unknown's avatar
unknown committed
1554
      !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
unknown's avatar
unknown committed
1555
  {
1556
    mi_check_print_error(param, "Not enough memory for extra record");
unknown's avatar
unknown committed
1557 1558 1559 1560 1561
    goto err;
  }

  if (!rep_quick)
  {
1562 1563
    /* Get real path for data file */
    if ((new_file=my_raid_create(fn_format(param->temp_filename,
1564
					   share->data_file_name, "",
unknown's avatar
unknown committed
1565
					   DATA_TMP_EXT, 2+4),
unknown's avatar
unknown committed
1566 1567 1568 1569 1570 1571 1572 1573 1574 1575
				 0,param->tmpfile_createflag,
				 share->base.raid_type,
				 share->base.raid_chunks,
				 share->base.raid_chunksize,
				 MYF(0))) < 0)
    {
      mi_check_print_error(param,"Can't create new tempfile: '%s'",
			   param->temp_filename);
      goto err;
    }
1576 1577
    if (new_header_length &&
        filecopy(param,new_file,info->dfile,0L,new_header_length,
unknown's avatar
unknown committed
1578 1579 1580 1581 1582 1583 1584 1585 1586 1587
		 "datafile-header"))
      goto err;
    info->s->state.dellink= HA_OFFSET_ERROR;
    info->rec_cache.file=new_file;
    if (param->testflag & T_UNPACK)
    {
      share->options&= ~HA_OPTION_COMPRESS_RECORD;
      mi_int2store(share->state.header.options,share->options);
    }
  }
unknown's avatar
unknown committed
1588 1589 1590 1591 1592 1593
  sort_info.info=info;
  sort_info.param = param;
  sort_param.read_cache=param->read_cache;
  sort_param.pos=sort_param.max_pos=share->pack.header_length;
  sort_param.filepos=new_header_length;
  param->read_cache.end_of_file=sort_info.filelength=
unknown's avatar
unknown committed
1594
    my_seek(info->dfile,0L,MY_SEEK_END,MYF(0));
unknown's avatar
unknown committed
1595 1596
  sort_info.dupp=0;
  sort_param.fix_datafile= (my_bool) (! rep_quick);
1597
  sort_param.master=1;
unknown's avatar
unknown committed
1598
  sort_info.max_records= ~(ha_rows) 0;
unknown's avatar
unknown committed
1599

unknown's avatar
unknown committed
1600
  set_data_file_type(&sort_info, share);
unknown's avatar
unknown committed
1601 1602 1603
  del=info->state->del;
  info->state->records=info->state->del=share->state.split=0;
  info->state->empty=0;
1604 1605
  param->glob_crc=0;
  if (param->testflag & T_CALC_CHECKSUM)
1606
    sort_param.calc_checksum= 1;
1607

unknown's avatar
unknown committed
1608
  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1609

1610
  /* This function always recreates all enabled indexes. */
1611
  if (param->testflag & T_CREATE_MISSING_KEYS)
1612
    mi_set_all_keys_active(share->state.key_map, share->base.keys);
1613
  mi_drop_all_indexes(param, info, TRUE);
unknown's avatar
unknown committed
1614 1615

  lock_memory(param);			/* Everything is alloced */
1616 1617

  /* Re-create all keys, which are set in key_map. */
unknown's avatar
unknown committed
1618
  while (!(error=sort_get_next_record(&sort_param)))
unknown's avatar
unknown committed
1619
  {
1620
    if (writekeys(&sort_param))
unknown's avatar
unknown committed
1621
    {
1622 1623
      if (my_errno != HA_ERR_FOUND_DUPP_KEY)
	goto err;
1624
      DBUG_DUMP("record",(uchar*) sort_param.record,share->base.pack_reclength);
unknown's avatar
unknown committed
1625
      mi_check_print_info(param,"Duplicate key %2d for record at %10s against new record at %10s",
1626
			  info->errkey+1,
unknown's avatar
unknown committed
1627
			  llstr(sort_param.start_recpos,llbuff),
1628
			  llstr(info->dupp_key_pos,llbuff2));
unknown's avatar
unknown committed
1629 1630 1631
      if (param->testflag & T_VERBOSE)
      {
	VOID(_mi_make_key(info,(uint) info->errkey,info->lastkey,
unknown's avatar
unknown committed
1632
			  sort_param.record,0L));
unknown's avatar
unknown committed
1633 1634 1635
	_mi_print_key(stdout,share->keyinfo[info->errkey].seg,info->lastkey,
		      USE_WHOLE_KEY);
      }
unknown's avatar
unknown committed
1636
      sort_info.dupp++;
1637
      if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
unknown's avatar
unknown committed
1638
      {
unknown's avatar
unknown committed
1639 1640
        param->testflag|=T_RETRY_WITHOUT_QUICK;
	param->error_printed=1;
unknown's avatar
unknown committed
1641 1642 1643 1644
	goto err;
      }
      continue;
    }
unknown's avatar
unknown committed
1645
    if (sort_write_record(&sort_param))
unknown's avatar
unknown committed
1646 1647
      goto err;
  }
unknown's avatar
unknown committed
1648
  if (error > 0 || write_data_suffix(&sort_info, (my_bool)!rep_quick) ||
unknown's avatar
unknown committed
1649 1650 1651 1652 1653 1654 1655
      flush_io_cache(&info->rec_cache) || param->read_cache.error < 0)
    goto err;

  if (param->testflag & T_WRITE_LOOP)
  {
    VOID(fputs("          \r",stdout)); VOID(fflush(stdout));
  }
1656
  if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0)))
unknown's avatar
unknown committed
1657 1658 1659 1660 1661 1662 1663
  {
    mi_check_print_warning(param,
			   "Can't change size of indexfile, error: %d",
			   my_errno);
    goto err;
  }

unknown's avatar
unknown committed
1664
  if (rep_quick && del+sort_info.dupp != info->state->del)
unknown's avatar
unknown committed
1665 1666 1667 1668
  {
    mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
    mi_check_print_error(param,"Run recovery again without -q");
    got_error=1;
unknown's avatar
unknown committed
1669 1670
    param->retry_repair=1;
    param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
1671 1672
    goto err;
  }
1673 1674 1675 1676 1677 1678 1679 1680 1681 1682
  if (param->testflag & T_SAFE_REPAIR)
  {
    /* Don't repair if we loosed more than one row */
    if (info->state->records+1 < start_records)
    {
      info->state->records=start_records;
      got_error=1;
      goto err;
    }
  }
unknown's avatar
unknown committed
1683 1684 1685 1686 1687

  if (!rep_quick)
  {
    my_close(info->dfile,MYF(0));
    info->dfile=new_file;
unknown's avatar
unknown committed
1688
    info->state->data_file_length=sort_param.filepos;
unknown's avatar
unknown committed
1689 1690 1691
    share->state.version=(ulong) time((time_t*) 0);	/* Force reopen */
  }
  else
1692
  {
unknown's avatar
unknown committed
1693
    info->state->data_file_length=sort_param.max_pos;
1694
  }
unknown's avatar
unknown committed
1695
  if (param->testflag & T_CALC_CHECKSUM)
1696
    info->state->checksum=param->glob_crc;
unknown's avatar
unknown committed
1697 1698 1699 1700 1701

  if (!(param->testflag & T_SILENT))
  {
    if (start_records != info->state->records)
      printf("Data records: %s\n", llstr(info->state->records,llbuff));
unknown's avatar
unknown committed
1702
    if (sort_info.dupp)
unknown's avatar
unknown committed
1703 1704
      mi_check_print_warning(param,
			     "%s records have been removed",
unknown's avatar
unknown committed
1705
			     llstr(sort_info.dupp,llbuff));
unknown's avatar
unknown committed
1706 1707 1708 1709 1710 1711 1712 1713
  }

  got_error=0;
  /* If invoked by external program that uses thr_lock */
  if (&share->state.state != info->state)
    memcpy( &share->state.state, info->state, sizeof(*info->state));

err:
1714 1715 1716 1717 1718 1719 1720
  if (!got_error)
  {
    /* Replace the actual file with the temporary file */
    if (new_file >= 0)
    {
      my_close(new_file,MYF(0));
      info->dfile=new_file= -1;
unknown's avatar
unknown committed
1721
      if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
1722 1723 1724
			    DATA_TMP_EXT,
                            param->backup_time,
                            share->base.raid_chunks,
1725 1726
			    (param->testflag & T_BACKUP_DATA ?
			     MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
1727
	  mi_open_datafile(info,share,name,-1))
1728
	got_error=1;
1729 1730

      param->retry_repair= 0;
1731 1732
    }
  }
unknown's avatar
unknown committed
1733 1734 1735 1736
  if (got_error)
  {
    if (! param->error_printed)
      mi_check_print_error(param,"%d for record at pos %s",my_errno,
unknown's avatar
unknown committed
1737
		  llstr(sort_param.start_recpos,llbuff));
unknown's avatar
unknown committed
1738 1739 1740 1741 1742
    if (new_file >= 0)
    {
      VOID(my_close(new_file,MYF(0)));
      VOID(my_raid_delete(param->temp_filename,info->s->base.raid_chunks,
			  MYF(MY_WME)));
1743
      info->rec_cache.file=-1; /* don't flush data to new_file, it's closed */
unknown's avatar
unknown committed
1744
    }
1745
    mi_mark_crashed_on_repair(info);
unknown's avatar
unknown committed
1746
  }
1747 1748
  my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff),
                            MYF(MY_ALLOW_ZERO_PTR));
1749 1750
  my_free(mi_get_rec_buff_ptr(info, sort_param.record),
          MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
1751
  my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
1752 1753 1754
  VOID(end_io_cache(&param->read_cache));
  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
  VOID(end_io_cache(&info->rec_cache));
unknown's avatar
unknown committed
1755
  got_error|=flush_blocks(param, share->key_cache, share->kfile);
unknown's avatar
unknown committed
1756 1757 1758 1759
  if (!got_error && param->testflag & T_UNPACK)
  {
    share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
    share->pack.header_length=0;
unknown's avatar
unknown committed
1760
    share->data_file_type=sort_info.new_data_file_type;
unknown's avatar
unknown committed
1761
  }
1762 1763
  share->state.changed|= (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES |
			  STATE_NOT_ANALYZED);
unknown's avatar
unknown committed
1764
  DBUG_RETURN(got_error);
1765
}
unknown's avatar
unknown committed
1766 1767 1768 1769


/* Uppate keyfile when doing repair */

1770
static int writekeys(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
1771 1772
{
  register uint i;
1773 1774
  uchar    *key;
  MI_INFO  *info=   sort_param->sort_info->info;
1775
  uchar    *buff=   sort_param->record;
1776
  my_off_t filepos= sort_param->filepos;
unknown's avatar
unknown committed
1777 1778 1779 1780 1781
  DBUG_ENTER("writekeys");

  key=info->lastkey+info->s->base.max_key_length;
  for (i=0 ; i < info->s->base.keys ; i++)
  {
1782
    if (mi_is_key_active(info->s->state.key_map, i))
unknown's avatar
unknown committed
1783
    {
1784 1785
      if (info->s->keyinfo[i].flag & HA_FULLTEXT )
      {
1786
        if (_mi_ft_add(info, i, key, buff, filepos))
1787
	  goto err;
1788
      }
1789
#ifdef HAVE_SPATIAL
1790 1791 1792 1793 1794 1795
      else if (info->s->keyinfo[i].flag & HA_SPATIAL)
      {
	uint key_length=_mi_make_key(info,i,key,buff,filepos);
	if (rtree_insert(info, i, key, key_length))
	  goto err;
      }
1796
#endif /*HAVE_SPATIAL*/
1797 1798 1799
      else
      {
	uint key_length=_mi_make_key(info,i,key,buff,filepos);
1800 1801
	if (_mi_ck_write(info,i,key,key_length))
	  goto err;
1802
      }
unknown's avatar
unknown committed
1803 1804 1805 1806 1807 1808 1809 1810 1811 1812
    }
  }
  DBUG_RETURN(0);

 err:
  if (my_errno == HA_ERR_FOUND_DUPP_KEY)
  {
    info->errkey=(int) i;			/* This key was found */
    while ( i-- > 0 )
    {
1813
      if (mi_is_key_active(info->s->state.key_map, i))
unknown's avatar
unknown committed
1814
      {
1815 1816
	if (info->s->keyinfo[i].flag & HA_FULLTEXT)
        {
1817
          if (_mi_ft_del(info,i, key,buff,filepos))
1818
	    break;
1819 1820 1821 1822
        }
        else
	{
	  uint key_length=_mi_make_key(info,i,key,buff,filepos);
1823 1824
	  if (_mi_ck_delete(info,i,key,key_length))
	    break;
1825
	}
unknown's avatar
unknown committed
1826 1827 1828
      }
    }
  }
1829
  /* Remove checksum that was added to glob_crc in sort_get_next_record */
1830 1831
  if (sort_param->calc_checksum)
    sort_param->sort_info->param->glob_crc-= info->checksum;
unknown's avatar
unknown committed
1832 1833 1834 1835 1836 1837 1838
  DBUG_PRINT("error",("errno: %d",my_errno));
  DBUG_RETURN(-1);
} /* writekeys */


	/* Change all key-pointers that points to a records */

1839
int movepoint(register MI_INFO *info, uchar *record, my_off_t oldpos,
unknown's avatar
unknown committed
1840 1841 1842 1843 1844 1845 1846 1847 1848 1849
	      my_off_t newpos, uint prot_key)
{
  register uint i;
  uchar *key;
  uint key_length;
  DBUG_ENTER("movepoint");

  key=info->lastkey+info->s->base.max_key_length;
  for (i=0 ; i < info->s->base.keys; i++)
  {
1850
    if (i != prot_key && mi_is_key_active(info->s->state.key_map, i))
unknown's avatar
unknown committed
1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864
    {
      key_length=_mi_make_key(info,i,key,record,oldpos);
      if (info->s->keyinfo[i].flag & HA_NOSAME)
      {					/* Change pointer direct */
	uint nod_flag;
	MI_KEYDEF *keyinfo;
	keyinfo=info->s->keyinfo+i;
	if (_mi_search(info,keyinfo,key,USE_WHOLE_KEY,
		       (uint) (SEARCH_SAME | SEARCH_SAVE_BUFF),
		       info->s->state.key_root[i]))
	  DBUG_RETURN(-1);
	nod_flag=mi_test_if_nod(info->buff);
	_mi_dpointer(info,info->int_keypos-nod_flag-
		     info->s->rec_reflength,newpos);
1865 1866
	if (_mi_write_keypage(info,keyinfo,info->last_keypage,
                              DFLT_INIT_HITS,info->buff))
unknown's avatar
unknown committed
1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884
	  DBUG_RETURN(-1);
      }
      else
      {					/* Change old key to new */
	if (_mi_ck_delete(info,i,key,key_length))
	  DBUG_RETURN(-1);
	key_length=_mi_make_key(info,i,key,record,newpos);
	if (_mi_ck_write(info,i,key,key_length))
	  DBUG_RETURN(-1);
      }
    }
  }
  DBUG_RETURN(0);
} /* movepoint */


	/* Tell system that we want all memory for our cache */

1885
void lock_memory(HA_CHECK *param __attribute__((unused)))
unknown's avatar
unknown committed
1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900
{
#ifdef SUN_OS				/* Key-cacheing thrases on sun 4.1 */
  if (param->opt_lock_memory)
  {
    int success = mlockall(MCL_CURRENT);	/* or plock(DATLOCK); */
    if (geteuid() == 0 && success != 0)
      mi_check_print_warning(param,
			     "Failed to lock memory. errno %d",my_errno);
  }
#endif
} /* lock_memory */


	/* Flush all changed blocks to disk */

1901
int flush_blocks(HA_CHECK *param, KEY_CACHE *key_cache, File file)
unknown's avatar
unknown committed
1902
{
1903
  if (flush_key_blocks(key_cache, file, FLUSH_RELEASE))
unknown's avatar
unknown committed
1904 1905 1906 1907 1908
  {
    mi_check_print_error(param,"%d when trying to write bufferts",my_errno);
    return(1);
  }
  if (!param->using_global_keycache)
1909
    end_key_cache(key_cache,1);
unknown's avatar
unknown committed
1910 1911 1912 1913 1914 1915
  return 0;
} /* flush_blocks */


	/* Sort index for more efficent reads */

1916
int mi_sort_index(HA_CHECK *param, register MI_INFO *info, char * name)
unknown's avatar
unknown committed
1917 1918 1919 1920
{
  reg2 uint key;
  reg1 MI_KEYDEF *keyinfo;
  File new_file;
1921
  my_off_t index_pos[HA_MAX_POSSIBLE_KEY];
unknown's avatar
unknown committed
1922
  uint r_locks,w_locks;
1923
  int old_lock;
unknown's avatar
unknown committed
1924
  MYISAM_SHARE *share=info->s;
1925
  MI_STATE_INFO old_state;
1926
  DBUG_ENTER("mi_sort_index");
unknown's avatar
unknown committed
1927

1928 1929 1930 1931
  /* cannot sort index files with R-tree indexes */
  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
       key++,keyinfo++)
    if (keyinfo->key_alg == HA_KEY_ALG_RTREE)
unknown's avatar
unknown committed
1932
      DBUG_RETURN(0);
1933

unknown's avatar
unknown committed
1934 1935 1936
  if (!(param->testflag & T_SILENT))
    printf("- Sorting index for MyISAM-table '%s'\n",name);

1937 1938 1939 1940
  /* Get real path for index file */
  fn_format(param->temp_filename,name,"", MI_NAME_IEXT,2+4+32);
  if ((new_file=my_create(fn_format(param->temp_filename,param->temp_filename,
				    "", INDEX_TMP_EXT,2+4),
unknown's avatar
unknown committed
1941 1942 1943 1944 1945 1946
			  0,param->tmpfile_createflag,MYF(0))) <= 0)
  {
    mi_check_print_error(param,"Can't create new tempfile: '%s'",
			 param->temp_filename);
    DBUG_RETURN(-1);
  }
unknown's avatar
unknown committed
1947 1948
  if (filecopy(param, new_file,share->kfile,0L,
	       (ulong) share->base.keystart, "headerblock"))
unknown's avatar
unknown committed
1949 1950
    goto err;

unknown's avatar
unknown committed
1951 1952
  param->new_file_pos=share->base.keystart;
  for (key= 0,keyinfo= &share->keyinfo[0]; key < share->base.keys ;
unknown's avatar
unknown committed
1953 1954
       key++,keyinfo++)
  {
1955
    if (! mi_is_key_active(info->s->state.key_map, key))
unknown's avatar
unknown committed
1956 1957
      continue;

unknown's avatar
unknown committed
1958
    if (share->state.key_root[key] != HA_OFFSET_ERROR)
unknown's avatar
unknown committed
1959
    {
1960
      index_pos[key]=param->new_file_pos;	/* Write first block here */
unknown's avatar
unknown committed
1961
      if (sort_one_index(param,info,keyinfo,share->state.key_root[key],
unknown's avatar
unknown committed
1962 1963 1964 1965 1966 1967 1968 1969
			 new_file))
	goto err;
    }
    else
      index_pos[key]= HA_OFFSET_ERROR;		/* No blocks */
  }

  /* Flush key cache for this file if we are calling this outside myisamchk */
unknown's avatar
unknown committed
1970
  flush_key_blocks(share->key_cache,share->kfile, FLUSH_IGNORE_CHANGED);
unknown's avatar
unknown committed
1971

unknown's avatar
unknown committed
1972
  share->state.version=(ulong) time((time_t*) 0);
1973 1974 1975 1976 1977
  old_state= share->state;			/* save state if not stored */
  r_locks=   share->r_locks;
  w_locks=   share->w_locks;
  old_lock=  info->lock_type;

1978
	/* Put same locks as old file */
1979
  share->r_locks= share->w_locks= share->tot_locks= 0;
unknown's avatar
unknown committed
1980 1981 1982
  (void) _mi_writeinfo(info,WRITEINFO_UPDATE_KEYFILE);
  VOID(my_close(share->kfile,MYF(MY_WME)));
  share->kfile = -1;
1983
  VOID(my_close(new_file,MYF(MY_WME)));
1984 1985
  if (change_to_newfile(share->index_file_name,MI_NAME_IEXT,INDEX_TMP_EXT,
                        0, 0, MYF(0)) ||
unknown's avatar
unknown committed
1986
      mi_open_keyfile(share))
1987
    goto err2;
1988
  info->lock_type= F_UNLCK;			/* Force mi_readinfo to lock */
unknown's avatar
unknown committed
1989
  _mi_readinfo(info,F_WRLCK,0);			/* Will lock the table */
1990 1991 1992
  info->lock_type=  old_lock;
  share->r_locks=   r_locks;
  share->w_locks=   w_locks;
1993
  share->tot_locks= r_locks+w_locks;
1994
  share->state=     old_state;			/* Restore old state */
unknown's avatar
unknown committed
1995 1996 1997 1998 1999

  info->state->key_file_length=param->new_file_pos;
  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
  for (key=0 ; key < info->s->base.keys ; key++)
    info->s->state.key_root[key]=index_pos[key];
2000
  for (key=0 ; key < info->s->state.header.max_block_size_index ; key++)
unknown's avatar
unknown committed
2001 2002
    info->s->state.key_del[key]=  HA_OFFSET_ERROR;

2003
  info->s->state.changed&= ~STATE_NOT_SORTED_PAGES;
unknown's avatar
unknown committed
2004 2005 2006 2007
  DBUG_RETURN(0);

err:
  VOID(my_close(new_file,MYF(MY_WME)));
2008
err2:
unknown's avatar
unknown committed
2009 2010
  VOID(my_delete(param->temp_filename,MYF(MY_WME)));
  DBUG_RETURN(-1);
2011
} /* mi_sort_index */
unknown's avatar
unknown committed
2012 2013 2014 2015


	 /* Sort records recursive using one index */

2016
static int sort_one_index(HA_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
unknown's avatar
unknown committed
2017 2018
			  my_off_t pagepos, File new_file)
{
2019
  uint length,nod_flag,used_length, key_length;
unknown's avatar
unknown committed
2020
  uchar *buff,*keypos,*endpos;
2021
  uchar key[HA_MAX_POSSIBLE_KEY_BUFF];
unknown's avatar
unknown committed
2022 2023 2024 2025
  my_off_t new_page_pos,next_page;
  char llbuff[22];
  DBUG_ENTER("sort_one_index");

2026 2027
  /* cannot walk over R-tree indices */
  DBUG_ASSERT(keyinfo->key_alg != HA_KEY_ALG_RTREE);
unknown's avatar
unknown committed
2028 2029 2030 2031 2032
  new_page_pos=param->new_file_pos;
  param->new_file_pos+=keyinfo->block_length;

  if (!(buff=(uchar*) my_alloca((uint) keyinfo->block_length)))
  {
2033
    mi_check_print_error(param,"Not enough memory for key block");
unknown's avatar
unknown committed
2034 2035
    DBUG_RETURN(-1);
  }
2036
  if (!_mi_fetch_keypage(info,keyinfo,pagepos,DFLT_INIT_HITS,buff,0))
unknown's avatar
unknown committed
2037 2038 2039 2040 2041
  {
    mi_check_print_error(param,"Can't read key block from filepos: %s",
		llstr(pagepos,llbuff));
    goto err;
  }
2042
  if ((nod_flag=mi_test_if_nod(buff)) || keyinfo->flag & HA_FULLTEXT)
unknown's avatar
unknown committed
2043 2044 2045 2046 2047 2048 2049 2050 2051 2052
  {
    used_length=mi_getint(buff);
    keypos=buff+2+nod_flag;
    endpos=buff+used_length;
    for ( ;; )
    {
      if (nod_flag)
      {
	next_page=_mi_kpos(nod_flag,keypos);
	_mi_kpointer(info,keypos-nod_flag,param->new_file_pos); /* Save new pos */
2053
	if (sort_one_index(param,info,keyinfo,next_page,new_file))
unknown's avatar
unknown committed
2054
	{
unknown's avatar
unknown committed
2055
	  DBUG_PRINT("error",
unknown's avatar
unknown committed
2056
		     ("From page: %ld, keyoffset: %lu  used_length: %d",
unknown's avatar
unknown committed
2057 2058
		      (ulong) pagepos, (ulong) (keypos - buff),
		      (int) used_length));
2059
	  DBUG_DUMP("buff",(uchar*) buff,used_length);
unknown's avatar
unknown committed
2060 2061 2062 2063
	  goto err;
	}
      }
      if (keypos >= endpos ||
2064
	  (key_length=(*keyinfo->get_key)(keyinfo,nod_flag,&keypos,key)) == 0)
unknown's avatar
unknown committed
2065
	break;
2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082
      DBUG_ASSERT(keypos <= endpos);
      if (keyinfo->flag & HA_FULLTEXT)
      {
        uint off;
        int  subkeys;
        get_key_full_length_rdonly(off, key);
        subkeys=ft_sintXkorr(key+off);
        if (subkeys < 0)
        {
          next_page= _mi_dpos(info,0,key+key_length);
          _mi_dpointer(info,keypos-nod_flag-info->s->rec_reflength,
                       param->new_file_pos); /* Save new pos */
          if (sort_one_index(param,info,&info->s->ft2_keyinfo,
                             next_page,new_file))
            goto err;
        }
      }
unknown's avatar
unknown committed
2083 2084 2085 2086 2087
    }
  }

  /* Fill block with zero and write it to the new index file */
  length=mi_getint(buff);
2088 2089
  bzero((uchar*) buff+length,keyinfo->block_length-length);
  if (my_pwrite(new_file,(uchar*) buff,(uint) keyinfo->block_length,
unknown's avatar
unknown committed
2090 2091 2092 2093 2094
		new_page_pos,MYF(MY_NABP | MY_WAIT_IF_FULL)))
  {
    mi_check_print_error(param,"Can't write indexblock, error: %d",my_errno);
    goto err;
  }
2095
  my_afree((uchar*) buff);
unknown's avatar
unknown committed
2096 2097
  DBUG_RETURN(0);
err:
2098
  my_afree((uchar*) buff);
unknown's avatar
unknown committed
2099 2100 2101 2102
  DBUG_RETURN(1);
} /* sort_one_index */


2103 2104 2105 2106 2107 2108 2109 2110
	/*
	  Let temporary file replace old file.
	  This assumes that the new file was created in the same
	  directory as given by realpath(filename).
	  This will ensure that any symlinks that are used will still work.
	  Copy stats from old file to new file, deletes orignal and
	  changes new file name to old file name
	*/
unknown's avatar
unknown committed
2111 2112 2113

int change_to_newfile(const char * filename, const char * old_ext,
		      const char * new_ext,
2114
                      time_t backup_time,
2115 2116
		      uint raid_chunks __attribute__((unused)),
		      myf MyFlags)
unknown's avatar
unknown committed
2117 2118 2119 2120 2121 2122 2123
{
  char old_filename[FN_REFLEN],new_filename[FN_REFLEN];
#ifdef USE_RAID
  if (raid_chunks)
    return my_raid_redel(fn_format(old_filename,filename,"",old_ext,2+4),
			 fn_format(new_filename,filename,"",new_ext,2+4),
			 raid_chunks,
2124
			 MYF(MY_WME | MY_LINK_WARNING | MyFlags));
unknown's avatar
unknown committed
2125
#endif
2126 2127 2128 2129
  /* Get real path to filename */
  (void) fn_format(old_filename,filename,"",old_ext,2+4+32);
  return my_redel(old_filename,
		  fn_format(new_filename,old_filename,"",new_ext,2+4),
2130
                  backup_time, MYF(MY_WME | MY_LINK_WARNING | MyFlags));
unknown's avatar
unknown committed
2131 2132 2133 2134 2135 2136
} /* change_to_newfile */


	/* Locks a whole file */
	/* Gives an error-message if file can't be locked */

2137
int lock_file(HA_CHECK *param, File file, my_off_t start, int lock_type,
unknown's avatar
unknown committed
2138 2139 2140 2141
	      const char *filetype, const char *filename)
{
  if (my_lock(file,lock_type,start,F_TO_EOF,
	      param->testflag & T_WAIT_FOREVER ? MYF(MY_SEEK_NOT_DONE) :
2142
	      MYF(MY_SEEK_NOT_DONE |  MY_SHORT_WAIT)))
unknown's avatar
unknown committed
2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153
  {
    mi_check_print_error(param," %d when locking %s '%s'",my_errno,filetype,filename);
    param->error_printed=2;		/* Don't give that data is crashed */
    return 1;
  }
  return 0;
} /* lock_file */


	/* Copy a block between two files */

2154
int filecopy(HA_CHECK *param, File to,File from,my_off_t start,
unknown's avatar
unknown committed
2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169
	     my_off_t length, const char *type)
{
  char tmp_buff[IO_SIZE],*buff;
  ulong buff_length;
  DBUG_ENTER("filecopy");

  buff_length=(ulong) min(param->write_buffer_length,length);
  if (!(buff=my_malloc(buff_length,MYF(0))))
  {
    buff=tmp_buff; buff_length=IO_SIZE;
  }

  VOID(my_seek(from,start,MY_SEEK_SET,MYF(0)));
  while (length > buff_length)
  {
2170 2171
    if (my_read(from,(uchar*) buff,buff_length,MYF(MY_NABP)) ||
	my_write(to,(uchar*) buff,buff_length,param->myf_rw))
unknown's avatar
unknown committed
2172 2173 2174
      goto err;
    length-= buff_length;
  }
2175 2176
  if (my_read(from,(uchar*) buff,(uint) length,MYF(MY_NABP)) ||
      my_write(to,(uchar*) buff,(uint) length,param->myf_rw))
unknown's avatar
unknown committed
2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187
    goto err;
  if (buff != tmp_buff)
    my_free(buff,MYF(0));
  DBUG_RETURN(0);
err:
  if (buff != tmp_buff)
    my_free(buff,MYF(0));
  mi_check_print_error(param,"Can't copy %s to tempfile, error %d",
		       type,my_errno);
  DBUG_RETURN(1);
}
unknown's avatar
unknown committed
2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203


/*
  Repair table or given index using sorting

  SYNOPSIS
    mi_repair_by_sort()
    param		Repair parameters
    info		MyISAM handler to repair
    name		Name of table (for warnings)
    rep_quick		set to <> 0 if we should not change data file

  RESULT
    0	ok
    <>0	Error
*/
unknown's avatar
unknown committed
2204

2205
int mi_repair_by_sort(HA_CHECK *param, register MI_INFO *info,
unknown's avatar
unknown committed
2206 2207 2208 2209 2210 2211 2212 2213 2214 2215
		      const char * name, int rep_quick)
{
  int got_error;
  uint i;
  ulong length;
  ha_rows start_records;
  my_off_t new_header_length,del;
  File new_file;
  MI_SORT_PARAM sort_param;
  MYISAM_SHARE *share=info->s;
2216
  HA_KEYSEG *keyseg;
unknown's avatar
unknown committed
2217 2218
  ulong   *rec_per_key_part;
  char llbuff[22];
2219
  MI_SORT_INFO sort_info;
2220
  ulonglong UNINIT_VAR(key_map);
unknown's avatar
unknown committed
2221
  DBUG_ENTER("mi_repair_by_sort");
unknown's avatar
unknown committed
2222 2223 2224 2225 2226 2227 2228 2229

  start_records=info->state->records;
  got_error=1;
  new_file= -1;
  new_header_length=(param->testflag & T_UNPACK) ? 0 :
    share->pack.header_length;
  if (!(param->testflag & T_SILENT))
  {
2230
    printf("- recovering (with sort) MyISAM-table '%s'\n",name);
unknown's avatar
unknown committed
2231 2232
    printf("Data records: %s\n", llstr(start_records,llbuff));
  }
2233
  param->testflag|=T_REP; /* for easy checking */
2234

2235 2236 2237
  if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
    param->testflag|=T_CALC_CHECKSUM;

unknown's avatar
unknown committed
2238
  bzero((char*)&sort_info,sizeof(sort_info));
2239
  bzero((char *)&sort_param, sizeof(sort_param));
unknown's avatar
unknown committed
2240
  if (!(sort_info.key_block=
unknown's avatar
unknown committed
2241 2242 2243 2244 2245 2246 2247 2248 2249 2250
	alloc_key_blocks(param,
			 (uint) param->sort_key_blocks,
			 share->base.max_key_block_length))
      || init_io_cache(&param->read_cache,info->dfile,
		       (uint) param->read_buffer_length,
		       READ_CACHE,share->pack.header_length,1,MYF(MY_WME)) ||
      (! rep_quick &&
       init_io_cache(&info->rec_cache,info->dfile,
		     (uint) param->write_buffer_length,
		     WRITE_CACHE,new_header_length,1,
2251
		     MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)))
unknown's avatar
unknown committed
2252
    goto err;
unknown's avatar
unknown committed
2253
  sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
unknown's avatar
unknown committed
2254 2255 2256
  info->opt_flag|=WRITE_CACHE_USED;
  info->rec_cache.file=info->dfile;		/* for sort_delete_record */

2257
  if (!mi_alloc_rec_buff(info, -1, &sort_param.record) ||
unknown's avatar
unknown committed
2258
      !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff))
unknown's avatar
unknown committed
2259
  {
2260
    mi_check_print_error(param, "Not enough memory for extra record");
unknown's avatar
unknown committed
2261 2262 2263 2264
    goto err;
  }
  if (!rep_quick)
  {
2265 2266
    /* Get real path for data file */
    if ((new_file=my_raid_create(fn_format(param->temp_filename,
2267 2268
					   share->data_file_name, "",
					   DATA_TMP_EXT, 2+4),
unknown's avatar
unknown committed
2269 2270 2271 2272 2273 2274 2275 2276 2277 2278
				 0,param->tmpfile_createflag,
				 share->base.raid_type,
				 share->base.raid_chunks,
				 share->base.raid_chunksize,
				 MYF(0))) < 0)
    {
      mi_check_print_error(param,"Can't create new tempfile: '%s'",
			   param->temp_filename);
      goto err;
    }
2279 2280
    if (new_header_length &&
        filecopy(param, new_file,info->dfile,0L,new_header_length,
unknown's avatar
unknown committed
2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292
		 "datafile-header"))
      goto err;
    if (param->testflag & T_UNPACK)
    {
      share->options&= ~HA_OPTION_COMPRESS_RECORD;
      mi_int2store(share->state.header.options,share->options);
    }
    share->state.dellink= HA_OFFSET_ERROR;
    info->rec_cache.file=new_file;
  }

  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2293 2294 2295 2296 2297

  /* Optionally drop indexes and optionally modify the key_map. */
  mi_drop_all_indexes(param, info, FALSE);
  key_map= share->state.key_map;
  if (param->testflag & T_CREATE_MISSING_KEYS)
unknown's avatar
unknown committed
2298
  {
2299 2300
    /* Invert the copied key_map to recreate all disabled indexes. */
    key_map= ~key_map;
unknown's avatar
unknown committed
2301
  }
unknown's avatar
unknown committed
2302

unknown's avatar
unknown committed
2303 2304
  sort_info.info=info;
  sort_info.param = param;
unknown's avatar
unknown committed
2305

unknown's avatar
unknown committed
2306 2307 2308 2309 2310
  set_data_file_type(&sort_info, share);
  sort_param.filepos=new_header_length;
  sort_info.dupp=0;
  sort_info.buff=0;
  param->read_cache.end_of_file=sort_info.filelength=
unknown's avatar
unknown committed
2311 2312
    my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));

unknown's avatar
unknown committed
2313
  sort_param.wordlist=NULL;
2314
  init_alloc_root(&sort_param.wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
2315

unknown's avatar
unknown committed
2316 2317 2318 2319 2320 2321
  if (share->data_file_type == DYNAMIC_RECORD)
    length=max(share->base.min_pack_length+1,share->base.min_block_length);
  else if (share->data_file_type == COMPRESSED_RECORD)
    length=share->base.min_block_length;
  else
    length=share->base.pack_reclength;
unknown's avatar
unknown committed
2322
  sort_info.max_records=
2323
    ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records :
unknown's avatar
unknown committed
2324
     (ha_rows) (sort_info.filelength/length+1));
unknown's avatar
unknown committed
2325 2326 2327
  sort_param.key_cmp=sort_key_cmp;
  sort_param.lock_in_memory=lock_memory;
  sort_param.tmpdir=param->tmpdir;
unknown's avatar
unknown committed
2328 2329
  sort_param.sort_info=&sort_info;
  sort_param.fix_datafile= (my_bool) (! rep_quick);
2330
  sort_param.master =1;
2331
  
unknown's avatar
unknown committed
2332
  del=info->state->del;
2333 2334
  param->glob_crc=0;
  if (param->testflag & T_CALC_CHECKSUM)
2335
    sort_param.calc_checksum= 1;
unknown's avatar
unknown committed
2336 2337

  rec_per_key_part= param->rec_per_key_part;
unknown's avatar
unknown committed
2338 2339
  for (sort_param.key=0 ; sort_param.key < share->base.keys ;
       rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++)
unknown's avatar
unknown committed
2340
  {
unknown's avatar
unknown committed
2341 2342
    sort_param.read_cache=param->read_cache;
    sort_param.keyinfo=share->keyinfo+sort_param.key;
2343
    sort_param.seg=sort_param.keyinfo->seg;
2344 2345 2346 2347
    /*
      Skip this index if it is marked disabled in the copied
      (and possibly inverted) key_map.
    */
2348
    if (! mi_is_key_active(key_map, sort_param.key))
2349 2350 2351
    {
      /* Remember old statistics for key */
      memcpy((char*) rec_per_key_part,
2352 2353
	     (char*) (share->state.rec_per_key_part +
		      (uint) (rec_per_key_part - param->rec_per_key_part)),
unknown's avatar
unknown committed
2354
	     sort_param.keyinfo->keysegs*sizeof(*rec_per_key_part));
2355 2356
      DBUG_PRINT("repair", ("skipping seemingly disabled index #: %u",
                            sort_param.key));
unknown's avatar
unknown committed
2357
      continue;
2358
    }
unknown's avatar
unknown committed
2359 2360

    if ((!(param->testflag & T_SILENT)))
unknown's avatar
unknown committed
2361 2362
      printf ("- Fixing index %d\n",sort_param.key+1);
    sort_param.max_pos=sort_param.pos=share->pack.header_length;
2363
    keyseg=sort_param.seg;
unknown's avatar
unknown committed
2364
    bzero((char*) sort_param.unique,sizeof(sort_param.unique));
unknown's avatar
unknown committed
2365
    sort_param.key_length=share->rec_reflength;
unknown's avatar
unknown committed
2366
    for (i=0 ; keyseg[i].type != HA_KEYTYPE_END; i++)
unknown's avatar
unknown committed
2367
    {
unknown's avatar
unknown committed
2368 2369 2370
      sort_param.key_length+=keyseg[i].length;
      if (keyseg[i].flag & HA_SPACE_PACK)
	sort_param.key_length+=get_pack_length(keyseg[i].length);
2371
      if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
unknown's avatar
unknown committed
2372 2373
	sort_param.key_length+=2 + test(keyseg[i].length >= 127);
      if (keyseg[i].flag & HA_NULL_PART)
unknown's avatar
unknown committed
2374 2375 2376 2377 2378
	sort_param.key_length++;
    }
    info->state->records=info->state->del=share->state.split=0;
    info->state->empty=0;

unknown's avatar
unknown committed
2379
    if (sort_param.keyinfo->flag & HA_FULLTEXT)
2380
    {
2381 2382
      uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
                                    sort_param.keyinfo->seg->charset->mbmaxlen;
2383 2384 2385 2386 2387
      sort_param.key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
      /*
        fulltext indexes may have much more entries than the
        number of rows in the table. We estimate the number here.
      */
2388
      if (sort_param.keyinfo->parser == &ft_default_parser)
2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404
      {
        /*
          for built-in parser the number of generated index entries
          cannot be larger than the size of the data file divided
          by the minimal word's length
        */
        sort_info.max_records=
          (ha_rows) (sort_info.filelength/ft_min_word_len+1);
      }
      else
      {
        /*
          for external plugin parser we cannot tell anything at all :(
          so, we'll use all the sort memory and start from ~10 buffpeks.
          (see _create_index_by_sort)
        */
2405 2406 2407
        sort_info.max_records= 10 *
                               max(param->sort_buffer_length, MIN_SORT_BUFFER) /
                               sort_param.key_length;
2408
      }
2409 2410

      sort_param.key_read=sort_ft_key_read;
2411
      sort_param.key_write=sort_ft_key_write;
2412 2413
    }
    else
2414
    {
2415
      sort_param.key_read=sort_key_read;
2416 2417
      sort_param.key_write=sort_key_write;
    }
2418

unknown's avatar
unknown committed
2419 2420 2421
    if (_create_index_by_sort(&sort_param,
			      (my_bool) (!(param->testflag & T_VERBOSE)),
			      (uint) param->sort_buffer_length))
2422 2423
    {
      param->retry_repair=1;
unknown's avatar
unknown committed
2424
      goto err;
2425
    }
2426 2427
    /* No need to calculate checksum again. */
    sort_param.calc_checksum= 0;
2428
    free_root(&sort_param.wordroot, MYF(0));
unknown's avatar
unknown committed
2429 2430

    /* Set for next loop */
unknown's avatar
unknown committed
2431
    sort_info.max_records= (ha_rows) info->state->records;
unknown's avatar
unknown committed
2432 2433

    if (param->testflag & T_STATISTICS)
unknown's avatar
unknown committed
2434
      update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique,
2435
                       param->stats_method == MI_STATS_METHOD_IGNORE_NULLS?
2436 2437 2438
                       sort_param.notnull: NULL,
                       (ulonglong) info->state->records);
    /* Enable this index in the permanent (not the copied) key_map. */
2439
    mi_set_key_active(share->state.key_map, sort_param.key);
2440
    DBUG_PRINT("repair", ("set enabled index #: %u", sort_param.key));
unknown's avatar
unknown committed
2441

unknown's avatar
unknown committed
2442
    if (sort_param.fix_datafile)
unknown's avatar
unknown committed
2443
    {
unknown's avatar
unknown committed
2444 2445
      param->read_cache.end_of_file=sort_param.filepos;
      if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
unknown's avatar
unknown committed
2446
	goto err;
2447 2448 2449 2450 2451 2452 2453 2454 2455
      if (param->testflag & T_SAFE_REPAIR)
      {
	/* Don't repair if we loosed more than one row */
	if (info->state->records+1 < start_records)
	{
	  info->state->records=start_records;
	  goto err;
	}
      }
unknown's avatar
unknown committed
2456 2457
      share->state.state.data_file_length = info->state->data_file_length=
	sort_param.filepos;
unknown's avatar
unknown committed
2458 2459 2460 2461
      /* Only whole records */
      share->state.version=(ulong) time((time_t*) 0);
      my_close(info->dfile,MYF(0));
      info->dfile=new_file;
unknown's avatar
unknown committed
2462
      share->data_file_type=sort_info.new_data_file_type;
unknown's avatar
unknown committed
2463
      share->pack.header_length=(ulong) new_header_length;
unknown's avatar
unknown committed
2464
      sort_param.fix_datafile=0;
unknown's avatar
unknown committed
2465 2466
    }
    else
unknown's avatar
unknown committed
2467
      info->state->data_file_length=sort_param.max_pos;
unknown's avatar
unknown committed
2468 2469

    param->read_cache.file=info->dfile;		/* re-init read cache */
unknown's avatar
unknown committed
2470 2471
    reinit_io_cache(&param->read_cache,READ_CACHE,share->pack.header_length,
                    1,1);
unknown's avatar
unknown committed
2472 2473 2474 2475 2476 2477 2478
  }

  if (param->testflag & T_WRITE_LOOP)
  {
    VOID(fputs("          \r",stdout)); VOID(fflush(stdout));
  }

unknown's avatar
unknown committed
2479
  if (rep_quick && del+sort_info.dupp != info->state->del)
unknown's avatar
unknown committed
2480 2481 2482 2483
  {
    mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
    mi_check_print_error(param,"Run recovery again without -q");
    got_error=1;
unknown's avatar
unknown committed
2484 2485
    param->retry_repair=1;
    param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
2486 2487 2488
    goto err;
  }

2489
  if (rep_quick && (param->testflag & T_FORCE_UNIQUENESS))
unknown's avatar
unknown committed
2490 2491 2492 2493 2494 2495 2496 2497 2498
  {
    my_off_t skr=info->state->data_file_length+
      (share->options & HA_OPTION_COMPRESS_RECORD ?
       MEMMAP_EXTRA_MARGIN : 0);
#ifdef USE_RELOC
    if (share->data_file_type == STATIC_RECORD &&
	skr < share->base.reloc*share->base.min_pack_length)
      skr=share->base.reloc*share->base.min_pack_length;
#endif
unknown's avatar
unknown committed
2499
    if (skr != sort_info.filelength && !info->s->base.raid_type)
2500
      if (my_chsize(info->dfile,skr,0,MYF(0)))
unknown's avatar
unknown committed
2501 2502 2503 2504
	mi_check_print_warning(param,
			       "Can't change size of datafile,  error: %d",
			       my_errno);
  }
unknown's avatar
unknown committed
2505
  if (param->testflag & T_CALC_CHECKSUM)
2506
    info->state->checksum=param->glob_crc;
2507

2508
  if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0)))
unknown's avatar
unknown committed
2509 2510 2511 2512 2513 2514 2515 2516
    mi_check_print_warning(param,
			   "Can't change size of indexfile, error: %d",
			   my_errno);

  if (!(param->testflag & T_SILENT))
  {
    if (start_records != info->state->records)
      printf("Data records: %s\n", llstr(info->state->records,llbuff));
unknown's avatar
unknown committed
2517
    if (sort_info.dupp)
unknown's avatar
unknown committed
2518 2519
      mi_check_print_warning(param,
			     "%s records have been removed",
unknown's avatar
unknown committed
2520
			     llstr(sort_info.dupp,llbuff));
unknown's avatar
unknown committed
2521 2522 2523 2524 2525 2526 2527
  }
  got_error=0;

  if (&share->state.state != info->state)
    memcpy( &share->state.state, info->state, sizeof(*info->state));

err:
unknown's avatar
unknown committed
2528
  got_error|= flush_blocks(param, share->key_cache, share->kfile);
2529 2530 2531 2532 2533 2534 2535 2536
  VOID(end_io_cache(&info->rec_cache));
  if (!got_error)
  {
    /* Replace the actual file with the temporary file */
    if (new_file >= 0)
    {
      my_close(new_file,MYF(0));
      info->dfile=new_file= -1;
unknown's avatar
unknown committed
2537
      if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
2538 2539
			    DATA_TMP_EXT, param->backup_time,
                            share->base.raid_chunks,
2540 2541
			    (param->testflag & T_BACKUP_DATA ?
			     MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
2542
	  mi_open_datafile(info,share,name,-1))
2543 2544 2545
	got_error=1;
    }
  }
unknown's avatar
unknown committed
2546 2547 2548 2549 2550 2551 2552
  if (got_error)
  {
    if (! param->error_printed)
      mi_check_print_error(param,"%d when fixing table",my_errno);
    if (new_file >= 0)
    {
      VOID(my_close(new_file,MYF(0)));
2553
      VOID(my_raid_delete(param->temp_filename,share->base.raid_chunks,
unknown's avatar
unknown committed
2554
			  MYF(MY_WME)));
2555 2556 2557
      if (info->dfile == new_file) /* Retry with key cache */
        if (unlikely(mi_open_datafile(info, share, name, -1)))
          param->retry_repair= 0; /* Safety */
unknown's avatar
unknown committed
2558
    }
2559
    mi_mark_crashed_on_repair(info);
unknown's avatar
unknown committed
2560
  }
2561 2562 2563 2564
  else if (key_map == share->state.key_map)
    share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
  share->state.changed|=STATE_NOT_SORTED_PAGES;

2565 2566
  my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff),
                            MYF(MY_ALLOW_ZERO_PTR));
2567 2568
  my_free(mi_get_rec_buff_ptr(info, sort_param.record),
          MYF(MY_ALLOW_ZERO_PTR));
2569 2570
  my_free((uchar*) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR));
  my_free((uchar*) sort_info.ft_buf, MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
2571
  my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
2572 2573
  VOID(end_io_cache(&param->read_cache));
  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
2574
  if (!got_error && (param->testflag & T_UNPACK))
unknown's avatar
unknown committed
2575 2576 2577 2578 2579
  {
    share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
    share->pack.header_length=0;
  }
  DBUG_RETURN(got_error);
2580
}
unknown's avatar
unknown committed
2581

unknown's avatar
unknown committed
2582 2583 2584 2585
/*
  Threaded repair of table using sorting

  SYNOPSIS
2586
    mi_repair_parallel()
unknown's avatar
unknown committed
2587 2588 2589 2590 2591 2592 2593 2594
    param		Repair parameters
    info		MyISAM handler to repair
    name		Name of table (for warnings)
    rep_quick		set to <> 0 if we should not change data file

  DESCRIPTION
    Same as mi_repair_by_sort but do it multithreaded
    Each key is handled by a separate thread.
unknown's avatar
unknown committed
2595
    TODO: make a number of threads a parameter
unknown's avatar
unknown committed
2596

2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618
    In parallel repair we use one thread per index. There are two modes:

    Quick

      Only the indexes are rebuilt. All threads share a read buffer.
      Every thread that needs fresh data in the buffer enters the shared
      cache lock. The last thread joining the lock reads the buffer from
      the data file and wakes all other threads.

    Non-quick

      The data file is rebuilt and all indexes are rebuilt to point to
      the new record positions. One thread is the master thread. It
      reads from the old data file and writes to the new data file. It
      also creates one of the indexes. The other threads read from a
      buffer which is filled by the master. If they need fresh data,
      they enter the shared cache lock. If the masters write buffer is
      full, it flushes it to the new data file and enters the shared
      cache lock too. When all threads joined in the lock, the master
      copies its write buffer to the read buffer for the other threads
      and wakes them.

unknown's avatar
unknown committed
2619 2620 2621 2622
  RESULT
    0	ok
    <>0	Error
*/
unknown's avatar
unknown committed
2623

2624
int mi_repair_parallel(HA_CHECK *param, register MI_INFO *info,
unknown's avatar
unknown committed
2625
			const char * name, int rep_quick)
unknown's avatar
unknown committed
2626
{
2627 2628 2629
#ifndef THREAD
  return mi_repair_by_sort(param, info, name, rep_quick);
#else
unknown's avatar
unknown committed
2630
  int got_error;
unknown's avatar
unknown committed
2631
  uint i,key, total_key_length, istep;
unknown's avatar
unknown committed
2632
  ulong rec_length;
unknown's avatar
unknown committed
2633 2634 2635
  ha_rows start_records;
  my_off_t new_header_length,del;
  File new_file;
unknown's avatar
unknown committed
2636
  MI_SORT_PARAM *sort_param=0;
unknown's avatar
unknown committed
2637 2638
  MYISAM_SHARE *share=info->s;
  ulong   *rec_per_key_part;
2639
  HA_KEYSEG *keyseg;
unknown's avatar
unknown committed
2640
  char llbuff[22];
2641
  IO_CACHE new_data_cache; /* For non-quick repair. */
unknown's avatar
unknown committed
2642
  IO_CACHE_SHARE io_share;
2643
  MI_SORT_INFO sort_info;
2644
  ulonglong UNINIT_VAR(key_map);
2645
  pthread_attr_t thr_attr;
2646
  ulong max_pack_reclength;
2647
  DBUG_ENTER("mi_repair_parallel");
unknown's avatar
unknown committed
2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658

  start_records=info->state->records;
  got_error=1;
  new_file= -1;
  new_header_length=(param->testflag & T_UNPACK) ? 0 :
    share->pack.header_length;
  if (!(param->testflag & T_SILENT))
  {
    printf("- parallel recovering (with sort) MyISAM-table '%s'\n",name);
    printf("Data records: %s\n", llstr(start_records,llbuff));
  }
2659
  param->testflag|=T_REP; /* for easy checking */
unknown's avatar
unknown committed
2660

2661 2662 2663
  if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
    param->testflag|=T_CALC_CHECKSUM;

2664 2665 2666
  /*
    Quick repair (not touching data file, rebuilding indexes):
    {
unknown's avatar
unknown committed
2667
      Read  cache is (HA_CHECK *param)->read_cache using info->dfile.
2668 2669 2670 2671 2672 2673
    }

    Non-quick repair (rebuilding data file and indexes):
    {
      Master thread:

unknown's avatar
unknown committed
2674
        Read  cache is (HA_CHECK *param)->read_cache using info->dfile.
2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692
        Write cache is (MI_INFO   *info)->rec_cache  using new_file.

      Slave threads:

        Read  cache is new_data_cache synced to master rec_cache.

      The final assignment of the filedescriptor for rec_cache is done
      after the cache creation.

      Don't check file size on new_data_cache, as the resulting file size
      is not known yet.

      As rec_cache and new_data_cache are synced, write_buffer_length is
      used for the read cache 'new_data_cache'. Both start at the same
      position 'new_header_length'.
    }
  */
  DBUG_PRINT("info", ("is quick repair: %d", rep_quick));
unknown's avatar
unknown committed
2693
  bzero((char*)&sort_info,sizeof(sort_info));
2694 2695 2696
  /* Initialize pthread structures before goto err. */
  pthread_mutex_init(&sort_info.mutex, MY_MUTEX_INIT_FAST);
  pthread_cond_init(&sort_info.cond, 0);
Sergey Vojtovich's avatar
Sergey Vojtovich committed
2697 2698
  pthread_mutex_init(&param->print_msg_mutex, MY_MUTEX_INIT_FAST);
  param->need_print_msg_lock= 1;
2699

unknown's avatar
unknown committed
2700
  if (!(sort_info.key_block=
2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714
	alloc_key_blocks(param, (uint) param->sort_key_blocks,
			 share->base.max_key_block_length)) ||
      init_io_cache(&param->read_cache, info->dfile,
                    (uint) param->read_buffer_length,
                    READ_CACHE, share->pack.header_length, 1, MYF(MY_WME)) ||
      (!rep_quick &&
       (init_io_cache(&info->rec_cache, info->dfile,
                      (uint) param->write_buffer_length,
                      WRITE_CACHE, new_header_length, 1,
                      MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw) ||
        init_io_cache(&new_data_cache, -1,
                      (uint) param->write_buffer_length,
                      READ_CACHE, new_header_length, 1,
                      MYF(MY_WME | MY_DONT_CHECK_FILESIZE)))))
unknown's avatar
unknown committed
2715 2716 2717
    goto err;
  sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks;
  info->opt_flag|=WRITE_CACHE_USED;
2718
  info->rec_cache.file=info->dfile;         /* for sort_delete_record */
unknown's avatar
unknown committed
2719 2720 2721 2722 2723

  if (!rep_quick)
  {
    /* Get real path for data file */
    if ((new_file=my_raid_create(fn_format(param->temp_filename,
2724
					   share->data_file_name, "",
unknown's avatar
unknown committed
2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736
					   DATA_TMP_EXT,
					   2+4),
				 0,param->tmpfile_createflag,
				 share->base.raid_type,
				 share->base.raid_chunks,
				 share->base.raid_chunksize,
				 MYF(0))) < 0)
    {
      mi_check_print_error(param,"Can't create new tempfile: '%s'",
			   param->temp_filename);
      goto err;
    }
2737 2738
    if (new_header_length &&
        filecopy(param, new_file,info->dfile,0L,new_header_length,
unknown's avatar
unknown committed
2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750
		 "datafile-header"))
      goto err;
    if (param->testflag & T_UNPACK)
    {
      share->options&= ~HA_OPTION_COMPRESS_RECORD;
      mi_int2store(share->state.header.options,share->options);
    }
    share->state.dellink= HA_OFFSET_ERROR;
    info->rec_cache.file=new_file;
  }

  info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2751 2752 2753 2754 2755

  /* Optionally drop indexes and optionally modify the key_map. */
  mi_drop_all_indexes(param, info, FALSE);
  key_map= share->state.key_map;
  if (param->testflag & T_CREATE_MISSING_KEYS)
unknown's avatar
unknown committed
2756
  {
2757 2758
    /* Invert the copied key_map to recreate all disabled indexes. */
    key_map= ~key_map;
unknown's avatar
unknown committed
2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770
  }

  sort_info.info=info;
  sort_info.param = param;

  set_data_file_type(&sort_info, share);
  sort_info.dupp=0;
  sort_info.buff=0;
  param->read_cache.end_of_file=sort_info.filelength=
    my_seek(param->read_cache.file,0L,MY_SEEK_END,MYF(0));

  if (share->data_file_type == DYNAMIC_RECORD)
unknown's avatar
unknown committed
2771
    rec_length=max(share->base.min_pack_length+1,share->base.min_block_length);
unknown's avatar
unknown committed
2772
  else if (share->data_file_type == COMPRESSED_RECORD)
unknown's avatar
unknown committed
2773
    rec_length=share->base.min_block_length;
unknown's avatar
unknown committed
2774
  else
unknown's avatar
unknown committed
2775
    rec_length=share->base.pack_reclength;
2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787
  /*
    +1 below is required hack for parallel repair mode.
    The info->state->records value, that is compared later
    to sort_info.max_records and cannot exceed it, is
    increased in sort_key_write. In mi_repair_by_sort, sort_key_write
    is called after sort_key_read, where the comparison is performed,
    but in parallel mode master thread can call sort_key_write
    before some other repair thread calls sort_key_read.
    Furthermore I'm not even sure +1 would be enough.
    May be sort_info.max_records shold be always set to max value in
    parallel mode.
  */
unknown's avatar
unknown committed
2788
  sort_info.max_records=
2789
    ((param->testflag & T_CREATE_MISSING_KEYS) ? info->state->records + 1:
unknown's avatar
unknown committed
2790
     (ha_rows) (sort_info.filelength/rec_length+1));
unknown's avatar
unknown committed
2791 2792 2793

  del=info->state->del;
  param->glob_crc=0;
2794 2795 2796 2797
  /* for compressed tables */
  max_pack_reclength= share->base.pack_reclength;
  if (share->options & HA_OPTION_COMPRESS_RECORD)
    set_if_bigger(max_pack_reclength, share->max_pack_length);
unknown's avatar
unknown committed
2798 2799
  if (!(sort_param=(MI_SORT_PARAM *)
        my_malloc((uint) share->base.keys *
2800
		  (sizeof(MI_SORT_PARAM) + max_pack_reclength),
unknown's avatar
unknown committed
2801
		  MYF(MY_ZEROFILL))))
unknown's avatar
unknown committed
2802
  {
2803
    mi_check_print_error(param,"Not enough memory for key!");
unknown's avatar
unknown committed
2804 2805
    goto err;
  }
unknown's avatar
unknown committed
2806
  total_key_length=0;
unknown's avatar
unknown committed
2807 2808 2809 2810
  rec_per_key_part= param->rec_per_key_part;
  info->state->records=info->state->del=share->state.split=0;
  info->state->empty=0;

unknown's avatar
unknown committed
2811 2812
  for (i=key=0, istep=1 ; key < share->base.keys ;
       rec_per_key_part+=sort_param[i].keyinfo->keysegs, i+=istep, key++)
unknown's avatar
unknown committed
2813 2814 2815
  {
    sort_param[i].key=key;
    sort_param[i].keyinfo=share->keyinfo+key;
2816
    sort_param[i].seg=sort_param[i].keyinfo->seg;
2817 2818 2819 2820
    /*
      Skip this index if it is marked disabled in the copied
      (and possibly inverted) key_map.
    */
2821
    if (! mi_is_key_active(key_map, key))
unknown's avatar
unknown committed
2822 2823 2824
    {
      /* Remember old statistics for key */
      memcpy((char*) rec_per_key_part,
2825 2826
	     (char*) (share->state.rec_per_key_part+
		      (uint) (rec_per_key_part - param->rec_per_key_part)),
unknown's avatar
unknown committed
2827
	     sort_param[i].keyinfo->keysegs*sizeof(*rec_per_key_part));
unknown's avatar
unknown committed
2828
      istep=0;
unknown's avatar
unknown committed
2829 2830
      continue;
    }
unknown's avatar
unknown committed
2831
    istep=1;
unknown's avatar
unknown committed
2832 2833
    if ((!(param->testflag & T_SILENT)))
      printf ("- Fixing index %d\n",key+1);
2834 2835 2836 2837 2838 2839 2840 2841 2842 2843
    if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
    {
      sort_param[i].key_read=sort_ft_key_read;
      sort_param[i].key_write=sort_ft_key_write;
    }
    else
    {
      sort_param[i].key_read=sort_key_read;
      sort_param[i].key_write=sort_key_write;
    }
unknown's avatar
unknown committed
2844 2845 2846 2847
    sort_param[i].key_cmp=sort_key_cmp;
    sort_param[i].lock_in_memory=lock_memory;
    sort_param[i].tmpdir=param->tmpdir;
    sort_param[i].sort_info=&sort_info;
2848
    sort_param[i].master=0;
unknown's avatar
unknown committed
2849
    sort_param[i].fix_datafile=0;
2850
    sort_param[i].calc_checksum= 0;
unknown's avatar
unknown committed
2851 2852 2853 2854

    sort_param[i].filepos=new_header_length;
    sort_param[i].max_pos=sort_param[i].pos=share->pack.header_length;

2855
    sort_param[i].record= (((uchar *)(sort_param+share->base.keys))+
2856
			   (max_pack_reclength * i));
unknown's avatar
unknown committed
2857 2858 2859 2860 2861
    if (!mi_alloc_rec_buff(info, -1, &sort_param[i].rec_buff))
    {
      mi_check_print_error(param,"Not enough memory!");
      goto err;
    }
unknown's avatar
unknown committed
2862 2863

    sort_param[i].key_length=share->rec_reflength;
2864
    for (keyseg=sort_param[i].seg; keyseg->type != HA_KEYTYPE_END;
unknown's avatar
unknown committed
2865
	 keyseg++)
unknown's avatar
unknown committed
2866 2867 2868 2869
    {
      sort_param[i].key_length+=keyseg->length;
      if (keyseg->flag & HA_SPACE_PACK)
        sort_param[i].key_length+=get_pack_length(keyseg->length);
2870
      if (keyseg->flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
unknown's avatar
unknown committed
2871 2872 2873 2874
        sort_param[i].key_length+=2 + test(keyseg->length >= 127);
      if (keyseg->flag & HA_NULL_PART)
        sort_param[i].key_length++;
    }
unknown's avatar
unknown committed
2875
    total_key_length+=sort_param[i].key_length;
unknown's avatar
unknown committed
2876 2877

    if (sort_param[i].keyinfo->flag & HA_FULLTEXT)
2878 2879 2880 2881
    {
      uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
                                    sort_param[i].keyinfo->seg->charset->mbmaxlen;
      sort_param[i].key_length+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
2882
      init_alloc_root(&sort_param[i].wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
2883
    }
unknown's avatar
unknown committed
2884 2885
  }
  sort_info.total_keys=i;
2886 2887
  sort_param[0].master= 1;
  sort_param[0].fix_datafile= (my_bool)(! rep_quick);
2888
  sort_param[0].calc_checksum= test(param->testflag & T_CALC_CHECKSUM);
unknown's avatar
unknown committed
2889

2890 2891 2892
  if (!ftparser_alloc_param(info))
    goto err;

unknown's avatar
unknown committed
2893
  sort_info.got_error=0;
unknown's avatar
unknown committed
2894
  pthread_mutex_lock(&sort_info.mutex);
unknown's avatar
unknown committed
2895

2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911
  /*
    Initialize the I/O cache share for use with the read caches and, in
    case of non-quick repair, the write cache. When all threads join on
    the cache lock, the writer copies the write cache contents to the
    read caches.
  */
  if (i > 1)
  {
    if (rep_quick)
      init_io_cache_share(&param->read_cache, &io_share, NULL, i);
    else
      init_io_cache_share(&new_data_cache, &io_share, &info->rec_cache, i);
  }
  else
    io_share.total_threads= 0; /* share not used */

2912 2913 2914
  (void) pthread_attr_init(&thr_attr);
  (void) pthread_attr_setdetachstate(&thr_attr,PTHREAD_CREATE_DETACHED);

unknown's avatar
unknown committed
2915
  for (i=0 ; i < sort_info.total_keys ; i++)
unknown's avatar
unknown committed
2916
  {
2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929
    /*
      Copy the properly initialized IO_CACHE structure so that every
      thread has its own copy. In quick mode param->read_cache is shared
      for use by all threads. In non-quick mode all threads but the
      first copy the shared new_data_cache, which is synchronized to the
      write cache of the first thread. The first thread copies
      param->read_cache, which is not shared.
    */
    sort_param[i].read_cache= ((rep_quick || !i) ? param->read_cache :
                               new_data_cache);
    DBUG_PRINT("io_cache_share", ("thread: %u  read_cache: 0x%lx",
                                  i, (long) &sort_param[i].read_cache));

unknown's avatar
unknown committed
2930 2931 2932 2933 2934 2935
    /*
      two approaches: the same amount of memory for each thread
      or the memory for the same number of keys for each thread...
      In the second one all the threads will fill their sort_buffers
      (and call write_keys) at the same time, putting more stress on i/o.
    */
unknown's avatar
unknown committed
2936
    sort_param[i].sortbuff_size=
unknown's avatar
unknown committed
2937
#ifndef USING_SECOND_APPROACH
unknown's avatar
unknown committed
2938 2939
      param->sort_buffer_length/sort_info.total_keys;
#else
unknown's avatar
unknown committed
2940
      param->sort_buffer_length*sort_param[i].key_length/total_key_length;
unknown's avatar
unknown committed
2941
#endif
2942 2943 2944
    if (pthread_create(&sort_param[i].thr, &thr_attr,
		       thr_find_all_keys,
		       (void *) (sort_param+i)))
unknown's avatar
unknown committed
2945 2946
    {
      mi_check_print_error(param,"Cannot start a repair thread");
2947 2948 2949 2950
      /* Cleanup: Detach from the share. Avoid others to be blocked. */
      if (io_share.total_threads)
        remove_io_thread(&sort_param[i].read_cache);
      DBUG_PRINT("error", ("Cannot start a repair thread"));
unknown's avatar
unknown committed
2951 2952 2953 2954 2955
      sort_info.got_error=1;
    }
    else
      sort_info.threads_running++;
  }
2956
  (void) pthread_attr_destroy(&thr_attr);
unknown's avatar
unknown committed
2957 2958 2959

  /* waiting for all threads to finish */
  while (sort_info.threads_running)
unknown's avatar
unknown committed
2960 2961
    pthread_cond_wait(&sort_info.cond, &sort_info.mutex);
  pthread_mutex_unlock(&sort_info.mutex);
unknown's avatar
unknown committed
2962

2963
  if ((got_error= thr_write_keys(sort_param)))
unknown's avatar
unknown committed
2964 2965 2966 2967
  {
    param->retry_repair=1;
    goto err;
  }
unknown's avatar
unknown committed
2968
  got_error=1;				/* Assume the following may go wrong */
unknown's avatar
unknown committed
2969

unknown's avatar
unknown committed
2970
  if (sort_param[0].fix_datafile)
unknown's avatar
unknown committed
2971
  {
2972 2973 2974 2975 2976
    /*
      Append some nuls to the end of a memory mapped file. Destroy the
      write cache. The master thread did already detach from the share
      by remove_io_thread() in sort.c:thr_find_all_keys().
    */
unknown's avatar
unknown committed
2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987
    if (write_data_suffix(&sort_info,1) || end_io_cache(&info->rec_cache))
      goto err;
    if (param->testflag & T_SAFE_REPAIR)
    {
      /* Don't repair if we loosed more than one row */
      if (info->state->records+1 < start_records)
      {
        info->state->records=start_records;
        goto err;
      }
    }
unknown's avatar
unknown committed
2988 2989
    share->state.state.data_file_length= info->state->data_file_length=
      sort_param->filepos;
unknown's avatar
unknown committed
2990 2991
    /* Only whole records */
    share->state.version=(ulong) time((time_t*) 0);
2992 2993 2994 2995 2996

    /*
      Exchange the data file descriptor of the table, so that we use the
      new file from now on.
     */
unknown's avatar
unknown committed
2997 2998
    my_close(info->dfile,MYF(0));
    info->dfile=new_file;
2999

unknown's avatar
unknown committed
3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014
    share->data_file_type=sort_info.new_data_file_type;
    share->pack.header_length=(ulong) new_header_length;
  }
  else
    info->state->data_file_length=sort_param->max_pos;

  if (rep_quick && del+sort_info.dupp != info->state->del)
  {
    mi_check_print_error(param,"Couldn't fix table with quick recovery: Found wrong number of deleted records");
    mi_check_print_error(param,"Run recovery again without -q");
    param->retry_repair=1;
    param->testflag|=T_RETRY_WITHOUT_QUICK;
    goto err;
  }

3015
  if (rep_quick && (param->testflag & T_FORCE_UNIQUENESS))
unknown's avatar
unknown committed
3016 3017 3018 3019 3020 3021 3022 3023 3024 3025
  {
    my_off_t skr=info->state->data_file_length+
      (share->options & HA_OPTION_COMPRESS_RECORD ?
       MEMMAP_EXTRA_MARGIN : 0);
#ifdef USE_RELOC
    if (share->data_file_type == STATIC_RECORD &&
	skr < share->base.reloc*share->base.min_pack_length)
      skr=share->base.reloc*share->base.min_pack_length;
#endif
    if (skr != sort_info.filelength && !info->s->base.raid_type)
3026
      if (my_chsize(info->dfile,skr,0,MYF(0)))
unknown's avatar
unknown committed
3027 3028 3029 3030 3031
	mi_check_print_warning(param,
			       "Can't change size of datafile,  error: %d",
			       my_errno);
  }
  if (param->testflag & T_CALC_CHECKSUM)
3032
    info->state->checksum=param->glob_crc;
unknown's avatar
unknown committed
3033

3034
  if (my_chsize(share->kfile,info->state->key_file_length,0,MYF(0)))
unknown's avatar
unknown committed
3035
    mi_check_print_warning(param,
unknown's avatar
unknown committed
3036
			   "Can't change size of indexfile, error: %d", my_errno);
unknown's avatar
unknown committed
3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049

  if (!(param->testflag & T_SILENT))
  {
    if (start_records != info->state->records)
      printf("Data records: %s\n", llstr(info->state->records,llbuff));
    if (sort_info.dupp)
      mi_check_print_warning(param,
			     "%s records have been removed",
			     llstr(sort_info.dupp,llbuff));
  }
  got_error=0;

  if (&share->state.state != info->state)
3050
    memcpy(&share->state.state, info->state, sizeof(*info->state));
unknown's avatar
unknown committed
3051 3052

err:
unknown's avatar
unknown committed
3053
  got_error|= flush_blocks(param, share->key_cache, share->kfile);
3054 3055 3056 3057 3058
  /*
    Destroy the write cache. The master thread did already detach from
    the share by remove_io_thread() or it was not yet started (if the
    error happend before creating the thread).
  */
unknown's avatar
unknown committed
3059
  VOID(end_io_cache(&info->rec_cache));
3060 3061 3062 3063 3064 3065 3066 3067
  /*
    Destroy the new data cache in case of non-quick repair. All slave
    threads did either detach from the share by remove_io_thread()
    already or they were not yet started (if the error happend before
    creating the threads).
  */
  if (!rep_quick)
    VOID(end_io_cache(&new_data_cache));
unknown's avatar
unknown committed
3068 3069 3070 3071 3072 3073 3074 3075
  if (!got_error)
  {
    /* Replace the actual file with the temporary file */
    if (new_file >= 0)
    {
      my_close(new_file,MYF(0));
      info->dfile=new_file= -1;
      if (change_to_newfile(share->data_file_name,MI_NAME_DEXT,
3076 3077
			    DATA_TMP_EXT, param->backup_time,
                            share->base.raid_chunks,
unknown's avatar
unknown committed
3078 3079
			    (param->testflag & T_BACKUP_DATA ?
			     MYF(MY_REDEL_MAKE_BACKUP): MYF(0))) ||
3080
	  mi_open_datafile(info,share,name,-1))
unknown's avatar
unknown committed
3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092
	got_error=1;
    }
  }
  if (got_error)
  {
    if (! param->error_printed)
      mi_check_print_error(param,"%d when fixing table",my_errno);
    if (new_file >= 0)
    {
      VOID(my_close(new_file,MYF(0)));
      VOID(my_raid_delete(param->temp_filename,share->base.raid_chunks,
			  MYF(MY_WME)));
3093 3094 3095
      if (info->dfile == new_file) /* Retry with key cache */
        if (unlikely(mi_open_datafile(info, share, name, -1)))
          param->retry_repair= 0; /* Safety */
unknown's avatar
unknown committed
3096 3097 3098 3099 3100 3101 3102
    }
    mi_mark_crashed_on_repair(info);
  }
  else if (key_map == share->state.key_map)
    share->state.changed&= ~STATE_NOT_OPTIMIZED_KEYS;
  share->state.changed|=STATE_NOT_SORTED_PAGES;

unknown's avatar
unknown committed
3103 3104
  pthread_cond_destroy (&sort_info.cond);
  pthread_mutex_destroy(&sort_info.mutex);
Sergey Vojtovich's avatar
Sergey Vojtovich committed
3105 3106
  pthread_mutex_destroy(&param->print_msg_mutex);
  param->need_print_msg_lock= 0;
unknown's avatar
unknown committed
3107

3108 3109 3110
  my_free((uchar*) sort_info.ft_buf, MYF(MY_ALLOW_ZERO_PTR));
  my_free((uchar*) sort_info.key_block,MYF(MY_ALLOW_ZERO_PTR));
  my_free((uchar*) sort_param,MYF(MY_ALLOW_ZERO_PTR));
unknown's avatar
unknown committed
3111 3112 3113 3114 3115 3116 3117 3118 3119
  my_free(sort_info.buff,MYF(MY_ALLOW_ZERO_PTR));
  VOID(end_io_cache(&param->read_cache));
  info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
  if (!got_error && (param->testflag & T_UNPACK))
  {
    share->state.header.options[0]&= (uchar) ~HA_OPTION_COMPRESS_RECORD;
    share->pack.header_length=0;
  }
  DBUG_RETURN(got_error);
3120
#endif /* THREAD */
unknown's avatar
unknown committed
3121
}
unknown's avatar
unknown committed
3122 3123 3124

	/* Read next record and return next key */

unknown's avatar
unknown committed
3125
static int sort_key_read(MI_SORT_PARAM *sort_param, void *key)
unknown's avatar
unknown committed
3126 3127
{
  int error;
3128
  MI_SORT_INFO *sort_info=sort_param->sort_info;
unknown's avatar
unknown committed
3129
  MI_INFO *info=sort_info->info;
unknown's avatar
unknown committed
3130 3131
  DBUG_ENTER("sort_key_read");

unknown's avatar
unknown committed
3132
  if ((error=sort_get_next_record(sort_param)))
unknown's avatar
unknown committed
3133 3134 3135 3136
    DBUG_RETURN(error);
  if (info->state->records == sort_info->max_records)
  {
    mi_check_print_error(sort_info->param,
3137 3138
			 "Key %d - Found too many records; Can't continue",
                         sort_param->key+1);
unknown's avatar
unknown committed
3139 3140
    DBUG_RETURN(1);
  }
unknown's avatar
unknown committed
3141
  sort_param->real_key_length=
unknown's avatar
unknown committed
3142
    (info->s->rec_reflength+
unknown's avatar
unknown committed
3143 3144
     _mi_make_key(info, sort_param->key, (uchar*) key,
		  sort_param->record, sort_param->filepos));
3145
#ifdef HAVE_valgrind
unknown's avatar
unknown committed
3146 3147 3148
  bzero(key+sort_param->real_key_length,
	(sort_param->key_length-sort_param->real_key_length));
#endif
unknown's avatar
unknown committed
3149
  DBUG_RETURN(sort_write_record(sort_param));
unknown's avatar
unknown committed
3150 3151
} /* sort_key_read */

unknown's avatar
unknown committed
3152
static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key)
3153 3154
{
  int error;
3155
  MI_SORT_INFO *sort_info=sort_param->sort_info;
unknown's avatar
unknown committed
3156
  MI_INFO *info=sort_info->info;
unknown's avatar
unknown committed
3157
  FT_WORD *wptr=0;
3158 3159
  DBUG_ENTER("sort_ft_key_read");

unknown's avatar
unknown committed
3160
  if (!sort_param->wordlist)
3161
  {
unknown's avatar
unknown committed
3162
    for (;;)
3163
    {
3164
      free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
unknown's avatar
unknown committed
3165
      if ((error=sort_get_next_record(sort_param)))
3166
        DBUG_RETURN(error);
3167 3168
      if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record,
                                    &sort_param->wordroot)))
3169
        DBUG_RETURN(1);
unknown's avatar
unknown committed
3170 3171
      if (wptr->pos)
        break;
unknown's avatar
unknown committed
3172
      error=sort_write_record(sort_param);
3173
    }
unknown's avatar
unknown committed
3174
    sort_param->wordptr=sort_param->wordlist=wptr;
3175 3176 3177 3178
  }
  else
  {
    error=0;
unknown's avatar
unknown committed
3179
    wptr=(FT_WORD*)(sort_param->wordptr);
3180 3181
  }

unknown's avatar
unknown committed
3182 3183 3184
  sort_param->real_key_length=(info->s->rec_reflength+
			       _ft_make_key(info, sort_param->key,
					    key, wptr++, sort_param->filepos));
3185
#ifdef HAVE_valgrind
unknown's avatar
unknown committed
3186 3187 3188 3189
  if (sort_param->key_length > sort_param->real_key_length)
    bzero(key+sort_param->real_key_length,
	  (sort_param->key_length-sort_param->real_key_length));
#endif
3190 3191
  if (!wptr->pos)
  {
3192
    free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
unknown's avatar
unknown committed
3193
    sort_param->wordlist=0;
unknown's avatar
unknown committed
3194
    error=sort_write_record(sort_param);
3195 3196
  }
  else
unknown's avatar
unknown committed
3197
    sort_param->wordptr=(void*)wptr;
3198 3199 3200

  DBUG_RETURN(error);
} /* sort_ft_key_read */
unknown's avatar
unknown committed
3201

unknown's avatar
unknown committed
3202

3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232
/*
  Read next record from file using parameters in sort_info.

  SYNOPSIS
    sort_get_next_record()
      sort_param                Information about and for the sort process

  NOTE

    Dynamic Records With Non-Quick Parallel Repair

      For non-quick parallel repair we use a synchronized read/write
      cache. This means that one thread is the master who fixes the data
      file by reading each record from the old data file and writing it
      to the new data file. By doing this the records in the new data
      file are written contiguously. Whenever the write buffer is full,
      it is copied to the read buffer. The slaves read from the read
      buffer, which is not associated with a file. Thus read_cache.file
      is -1. When using _mi_read_cache(), the slaves must always set
      flag to READING_NEXT so that the function never tries to read from
      file. This is safe because the records are contiguous. There is no
      need to read outside the cache. This condition is evaluated in the
      variable 'parallel_flag' for quick reference. read_cache.file must
      be >= 0 in every other case.

  RETURN
    -1          end of file
    0           ok
    > 0         error
*/
unknown's avatar
unknown committed
3233

unknown's avatar
unknown committed
3234
static int sort_get_next_record(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
3235 3236
{
  int searching;
3237
  int parallel_flag;
unknown's avatar
unknown committed
3238 3239
  uint found_record,b_type,left_length;
  my_off_t pos;
Staale Smedseng's avatar
Staale Smedseng committed
3240
  uchar *UNINIT_VAR(to);
unknown's avatar
unknown committed
3241
  MI_BLOCK_INFO block_info;
3242 3243
  MI_SORT_INFO *sort_info=sort_param->sort_info;
  HA_CHECK *param=sort_info->param;
3244 3245
  MI_INFO *info=sort_info->info;
  MYISAM_SHARE *share=info->s;
unknown's avatar
unknown committed
3246 3247 3248
  char llbuff[22],llbuff2[22];
  DBUG_ENTER("sort_get_next_record");

3249
  if (killed_ptr(param))
unknown's avatar
unknown committed
3250 3251
    DBUG_RETURN(1);

unknown's avatar
unknown committed
3252
  switch (share->data_file_type) {
unknown's avatar
unknown committed
3253 3254 3255
  case BLOCK_RECORD:
    DBUG_ASSERT(0);                           /* Impossible */
    break;
unknown's avatar
unknown committed
3256 3257 3258
  case STATIC_RECORD:
    for (;;)
    {
unknown's avatar
unknown committed
3259
      if (my_b_read(&sort_param->read_cache,sort_param->record,
unknown's avatar
unknown committed
3260
		    share->base.pack_reclength))
3261
      {
unknown's avatar
unknown committed
3262
	if (sort_param->read_cache.error)
3263
	  param->out_flag |= O_DATA_LOST;
unknown's avatar
unknown committed
3264 3265
        param->retry_repair=1;
        param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3266
	DBUG_RETURN(-1);
3267
      }
unknown's avatar
unknown committed
3268 3269
      sort_param->start_recpos=sort_param->pos;
      if (!sort_param->fix_datafile)
unknown's avatar
unknown committed
3270
      {
unknown's avatar
unknown committed
3271
	sort_param->filepos=sort_param->pos;
3272 3273
        if (sort_param->master)
	  share->state.split++;
unknown's avatar
unknown committed
3274
      }
unknown's avatar
unknown committed
3275 3276
      sort_param->max_pos=(sort_param->pos+=share->base.pack_reclength);
      if (*sort_param->record)
3277
      {
3278
	if (sort_param->calc_checksum)
3279
	  param->glob_crc+= (info->checksum=
3280 3281 3282
                             (*info->s->calc_check_checksum)(info,
                                                             sort_param->
                                                             record));
unknown's avatar
unknown committed
3283
	DBUG_RETURN(0);
3284
      }
3285
      if (!sort_param->fix_datafile && sort_param->master)
unknown's avatar
unknown committed
3286 3287 3288 3289 3290 3291 3292
      {
	info->state->del++;
	info->state->empty+=share->base.pack_reclength;
      }
    }
  case DYNAMIC_RECORD:
    LINT_INIT(to);
unknown's avatar
unknown committed
3293 3294
    pos=sort_param->pos;
    searching=(sort_param->fix_datafile && (param->testflag & T_EXTEND));
3295
    parallel_flag= (sort_param->read_cache.file < 0) ? READING_NEXT : 0;
unknown's avatar
unknown committed
3296 3297 3298 3299 3300
    for (;;)
    {
      found_record=block_info.second_read= 0;
      left_length=1;
      if (searching)
unknown's avatar
unknown committed
3301
      {
unknown's avatar
unknown committed
3302
	pos=MY_ALIGN(pos,MI_DYN_ALIGN_SIZE);
unknown's avatar
unknown committed
3303
        param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3304
	sort_param->start_recpos=pos;
unknown's avatar
unknown committed
3305
      }
unknown's avatar
unknown committed
3306 3307
      do
      {
unknown's avatar
unknown committed
3308 3309
	if (pos > sort_param->max_pos)
	  sort_param->max_pos=pos;
unknown's avatar
unknown committed
3310 3311 3312 3313 3314
	if (pos & (MI_DYN_ALIGN_SIZE-1))
	{
	  if ((param->testflag & T_VERBOSE) || searching == 0)
	    mi_check_print_info(param,"Wrong aligned block at %s",
				llstr(pos,llbuff));
unknown's avatar
unknown committed
3315
	  if (searching)
unknown's avatar
unknown committed
3316 3317 3318 3319 3320
	    goto try_next;
	}
	if (found_record && pos == param->search_after_block)
	  mi_check_print_info(param,"Block: %s used by record at %s",
		     llstr(param->search_after_block,llbuff),
unknown's avatar
unknown committed
3321 3322
		     llstr(sort_param->start_recpos,llbuff2));
	if (_mi_read_cache(&sort_param->read_cache,
3323
                           (uchar*) block_info.header,pos,
3324 3325
			   MI_BLOCK_INFO_HEADER_LENGTH,
			   (! found_record ? READING_NEXT : 0) |
3326
                           parallel_flag | READING_HEADER))
unknown's avatar
unknown committed
3327 3328 3329 3330 3331
	{
	  if (found_record)
	  {
	    mi_check_print_info(param,
				"Can't read whole record at %s (errno: %d)",
unknown's avatar
unknown committed
3332
				llstr(sort_param->start_recpos,llbuff),errno);
unknown's avatar
unknown committed
3333 3334 3335 3336
	    goto try_next;
	  }
	  DBUG_RETURN(-1);
	}
unknown's avatar
unknown committed
3337
	if (searching && ! sort_param->fix_datafile)
unknown's avatar
unknown committed
3338 3339
	{
	  param->error_printed=1;
unknown's avatar
unknown committed
3340 3341
          param->retry_repair=1;
          param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3342 3343
	  DBUG_RETURN(1);	/* Something wrong with data */
	}
3344 3345 3346
	b_type=_mi_get_block_info(&block_info,-1,pos);
	if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) ||
	   ((b_type & BLOCK_FIRST) &&
unknown's avatar
unknown committed
3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367
	     (block_info.rec_len < (uint) share->base.min_pack_length ||
	      block_info.rec_len > (uint) share->base.max_pack_length)))
	{
	  uint i;
	  if (param->testflag & T_VERBOSE || searching == 0)
	    mi_check_print_info(param,
				"Wrong bytesec: %3d-%3d-%3d at %10s; Skipped",
		       block_info.header[0],block_info.header[1],
		       block_info.header[2],llstr(pos,llbuff));
	  if (found_record)
	    goto try_next;
	  block_info.second_read=0;
	  searching=1;
	  /* Search after block in read header string */
	  for (i=MI_DYN_ALIGN_SIZE ;
	       i < MI_BLOCK_INFO_HEADER_LENGTH ;
	       i+= MI_DYN_ALIGN_SIZE)
	    if (block_info.header[i] >= 1 &&
		block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE)
	      break;
	  pos+=(ulong) i;
unknown's avatar
unknown committed
3368
	  sort_param->start_recpos=pos;
unknown's avatar
unknown committed
3369 3370 3371 3372
	  continue;
	}
	if (b_type & BLOCK_DELETED)
	{
3373
	  my_bool error=0;
unknown's avatar
unknown committed
3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402
	  if (block_info.block_len+ (uint) (block_info.filepos-pos) <
	      share->base.min_block_length)
	  {
	    if (!searching)
	      mi_check_print_info(param,
				  "Deleted block with impossible length %u at %s",
				  block_info.block_len,llstr(pos,llbuff));
	    error=1;
	  }
	  else
	  {
	    if ((block_info.next_filepos != HA_OFFSET_ERROR &&
		 block_info.next_filepos >=
		 info->state->data_file_length) ||
		(block_info.prev_filepos != HA_OFFSET_ERROR &&
		 block_info.prev_filepos >= info->state->data_file_length))
	    {
	      if (!searching)
		mi_check_print_info(param,
				    "Delete link points outside datafile at %s",
				    llstr(pos,llbuff));
	      error=1;
	    }
	  }
	  if (error)
	  {
	    if (found_record)
	      goto try_next;
	    searching=1;
3403
	    pos+= MI_DYN_ALIGN_SIZE;
unknown's avatar
unknown committed
3404
	    sort_param->start_recpos=pos;
unknown's avatar
unknown committed
3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423
	    block_info.second_read=0;
	    continue;
	  }
	}
	else
	{
	  if (block_info.block_len+ (uint) (block_info.filepos-pos) <
	      share->base.min_block_length ||
	      block_info.block_len > (uint) share->base.max_pack_length+
	      MI_SPLIT_LENGTH)
	  {
	    if (!searching)
	      mi_check_print_info(param,
				  "Found block with impossible length %u at %s; Skipped",
				  block_info.block_len+ (uint) (block_info.filepos-pos),
				  llstr(pos,llbuff));
	    if (found_record)
	      goto try_next;
	    searching=1;
3424
	    pos+= MI_DYN_ALIGN_SIZE;
unknown's avatar
unknown committed
3425
	    sort_param->start_recpos=pos;
unknown's avatar
unknown committed
3426 3427 3428 3429 3430 3431
	    block_info.second_read=0;
	    continue;
	  }
	}
	if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR))
	{
3432 3433
          if (!sort_param->fix_datafile && sort_param->master &&
              (b_type & BLOCK_DELETED))
unknown's avatar
unknown committed
3434 3435 3436 3437 3438 3439 3440 3441
	  {
	    info->state->empty+=block_info.block_len;
	    info->state->del++;
	    share->state.split++;
	  }
	  if (found_record)
	    goto try_next;
	  if (searching)
3442 3443
	  {
	    pos+=MI_DYN_ALIGN_SIZE;
unknown's avatar
unknown committed
3444
	    sort_param->start_recpos=pos;
3445
	  }
unknown's avatar
unknown committed
3446 3447 3448 3449 3450 3451
	  else
	    pos=block_info.filepos+block_info.block_len;
	  block_info.second_read=0;
	  continue;
	}

3452
	if (!sort_param->fix_datafile && sort_param->master)
unknown's avatar
unknown committed
3453
	  share->state.split++;
unknown's avatar
unknown committed
3454 3455
	if (! found_record++)
	{
unknown's avatar
unknown committed
3456 3457 3458 3459 3460 3461
	  sort_param->find_length=left_length=block_info.rec_len;
	  sort_param->start_recpos=pos;
	  if (!sort_param->fix_datafile)
	    sort_param->filepos=sort_param->start_recpos;
	  if (sort_param->fix_datafile && (param->testflag & T_EXTEND))
	    sort_param->pos=block_info.filepos+1;
unknown's avatar
unknown committed
3462
	  else
unknown's avatar
unknown committed
3463
	    sort_param->pos=block_info.filepos+block_info.block_len;
unknown's avatar
unknown committed
3464 3465
	  if (share->base.blobs)
	  {
3466
	    if (!(to=mi_alloc_rec_buff(info,block_info.rec_len,
3467
				       &(sort_param->rec_buff))))
unknown's avatar
unknown committed
3468
	    {
3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482
	      if (param->max_record_length >= block_info.rec_len)
	      {
		mi_check_print_error(param,"Not enough memory for blob at %s (need %lu)",
				     llstr(sort_param->start_recpos,llbuff),
				     (ulong) block_info.rec_len);
		DBUG_RETURN(1);
	      }
	      else
	      {
		mi_check_print_info(param,"Not enough memory for blob at %s (need %lu); Row skipped",
				    llstr(sort_param->start_recpos,llbuff),
				    (ulong) block_info.rec_len);
		goto try_next;
	      }
unknown's avatar
unknown committed
3483 3484 3485
	    }
	  }
	  else
3486
	    to= sort_param->rec_buff;
unknown's avatar
unknown committed
3487 3488 3489
	}
	if (left_length < block_info.data_len || ! block_info.data_len)
	{
3490 3491
	  mi_check_print_info(param,
			      "Found block with too small length at %s; Skipped",
unknown's avatar
unknown committed
3492
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3493 3494 3495
	  goto try_next;
	}
	if (block_info.filepos + block_info.data_len >
unknown's avatar
unknown committed
3496
	    sort_param->read_cache.end_of_file)
unknown's avatar
unknown committed
3497
	{
3498 3499
	  mi_check_print_info(param,
			      "Found block that points outside data file at %s",
unknown's avatar
unknown committed
3500
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3501 3502
	  goto try_next;
	}
3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527
        /*
          Copy information that is already read. Avoid accessing data
          below the cache start. This could happen if the header
          streched over the end of the previous buffer contents.
        */
        {
          uint header_len= (uint) (block_info.filepos - pos);
          uint prefetch_len= (MI_BLOCK_INFO_HEADER_LENGTH - header_len);

          if (prefetch_len > block_info.data_len)
            prefetch_len= block_info.data_len;
          if (prefetch_len)
          {
            memcpy(to, block_info.header + header_len, prefetch_len);
            block_info.filepos+= prefetch_len;
            block_info.data_len-= prefetch_len;
            left_length-= prefetch_len;
            to+= prefetch_len;
          }
        }
        if (block_info.data_len &&
            _mi_read_cache(&sort_param->read_cache,to,block_info.filepos,
                           block_info.data_len,
                           (found_record == 1 ? READING_NEXT : 0) |
                           parallel_flag))
unknown's avatar
unknown committed
3528
	{
3529 3530
	  mi_check_print_info(param,
			      "Read error for block at: %s (error: %d); Skipped",
3531
			      llstr(block_info.filepos,llbuff),my_errno);
unknown's avatar
unknown committed
3532 3533 3534 3535 3536 3537 3538 3539
	  goto try_next;
	}
	left_length-=block_info.data_len;
	to+=block_info.data_len;
	pos=block_info.next_filepos;
	if (pos == HA_OFFSET_ERROR && left_length)
	{
	  mi_check_print_info(param,"Wrong block with wrong total length starting at %s",
unknown's avatar
unknown committed
3540
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3541 3542
	  goto try_next;
	}
unknown's avatar
unknown committed
3543
	if (pos + MI_BLOCK_INFO_HEADER_LENGTH > sort_param->read_cache.end_of_file)
unknown's avatar
unknown committed
3544 3545
	{
	  mi_check_print_info(param,"Found link that points at %s (outside data file) at %s",
3546
			      llstr(pos,llbuff2),
unknown's avatar
unknown committed
3547
			      llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3548 3549 3550 3551
	  goto try_next;
	}
      } while (left_length);

3552
      if (_mi_rec_unpack(info,sort_param->record,sort_param->rec_buff,
unknown's avatar
unknown committed
3553
			 sort_param->find_length) != MY_FILE_ERROR)
unknown's avatar
unknown committed
3554
      {
unknown's avatar
unknown committed
3555
	if (sort_param->read_cache.error < 0)
unknown's avatar
unknown committed
3556
	  DBUG_RETURN(1);
3557
	if (sort_param->calc_checksum)
3558 3559
	  info->checksum= (*info->s->calc_check_checksum)(info,
                                                          sort_param->record);
unknown's avatar
unknown committed
3560 3561
	if ((param->testflag & (T_EXTEND | T_REP)) || searching)
	{
unknown's avatar
unknown committed
3562
	  if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff,
3563 3564
                            sort_param->find_length,
                            (param->testflag & T_QUICK) &&
3565
                            sort_param->calc_checksum &&
3566
                            test(info->s->calc_checksum)))
unknown's avatar
unknown committed
3567 3568
	  {
	    mi_check_print_info(param,"Found wrong packed record at %s",
unknown's avatar
unknown committed
3569
				llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3570 3571 3572
	    goto try_next;
	  }
	}
3573
	if (sort_param->calc_checksum)
3574
	  param->glob_crc+= info->checksum;
unknown's avatar
unknown committed
3575 3576
	DBUG_RETURN(0);
      }
unknown's avatar
unknown committed
3577
      if (!searching)
3578 3579 3580
        mi_check_print_info(param,"Key %d - Found wrong stored record at %s",
                            sort_param->key+1,
                            llstr(sort_param->start_recpos,llbuff));
unknown's avatar
unknown committed
3581
    try_next:
unknown's avatar
unknown committed
3582
      pos=(sort_param->start_recpos+=MI_DYN_ALIGN_SIZE);
unknown's avatar
unknown committed
3583 3584 3585
      searching=1;
    }
  case COMPRESSED_RECORD:
unknown's avatar
unknown committed
3586
    for (searching=0 ;; searching=1, sort_param->pos++)
unknown's avatar
unknown committed
3587
    {
3588
      if (_mi_read_cache(&sort_param->read_cache,(uchar*) block_info.header,
unknown's avatar
unknown committed
3589
			 sort_param->pos,
3590
			 share->pack.ref_length,READING_NEXT))
unknown's avatar
unknown committed
3591
	DBUG_RETURN(-1);
unknown's avatar
unknown committed
3592
      if (searching && ! sort_param->fix_datafile)
unknown's avatar
unknown committed
3593 3594
      {
	param->error_printed=1;
unknown's avatar
unknown committed
3595 3596
        param->retry_repair=1;
        param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3597 3598
	DBUG_RETURN(1);		/* Something wrong with data */
      }
unknown's avatar
unknown committed
3599
      sort_param->start_recpos=sort_param->pos;
3600 3601
      if (_mi_pack_get_block_info(info, &sort_param->bit_buff, &block_info,
                                  &sort_param->rec_buff, -1, sort_param->pos))
unknown's avatar
unknown committed
3602 3603
	DBUG_RETURN(-1);
      if (!block_info.rec_len &&
unknown's avatar
unknown committed
3604 3605
	  sort_param->pos + MEMMAP_EXTRA_MARGIN ==
	  sort_param->read_cache.end_of_file)
unknown's avatar
unknown committed
3606 3607 3608 3609 3610 3611
	DBUG_RETURN(-1);
      if (block_info.rec_len < (uint) share->min_pack_length ||
	  block_info.rec_len > (uint) share->max_pack_length)
      {
	if (! searching)
	  mi_check_print_info(param,"Found block with wrong recordlength: %d at %s\n",
3612
			      block_info.rec_len,
unknown's avatar
unknown committed
3613
			      llstr(sort_param->pos,llbuff));
unknown's avatar
unknown committed
3614 3615
	continue;
      }
3616
      if (_mi_read_cache(&sort_param->read_cache,(uchar*) sort_param->rec_buff,
3617 3618
			 block_info.filepos, block_info.rec_len,
			 READING_NEXT))
unknown's avatar
unknown committed
3619 3620
      {
	if (! searching)
3621
	  mi_check_print_info(param,"Couldn't read whole record from %s",
unknown's avatar
unknown committed
3622
			      llstr(sort_param->pos,llbuff));
unknown's avatar
unknown committed
3623 3624
	continue;
      }
3625 3626
      if (_mi_pack_rec_unpack(info, &sort_param->bit_buff, sort_param->record,
                              sort_param->rec_buff, block_info.rec_len))
unknown's avatar
unknown committed
3627 3628
      {
	if (! searching)
3629
	  mi_check_print_info(param,"Found wrong record at %s",
unknown's avatar
unknown committed
3630
			      llstr(sort_param->pos,llbuff));
unknown's avatar
unknown committed
3631 3632
	continue;
      }
unknown's avatar
unknown committed
3633
      if (!sort_param->fix_datafile)
unknown's avatar
unknown committed
3634
      {
unknown's avatar
unknown committed
3635
	sort_param->filepos=sort_param->pos;
3636 3637
        if (sort_param->master)
	  share->state.split++;
unknown's avatar
unknown committed
3638
      }
unknown's avatar
unknown committed
3639
      sort_param->max_pos=(sort_param->pos=block_info.filepos+
unknown's avatar
unknown committed
3640 3641
			 block_info.rec_len);
      info->packed_length=block_info.rec_len;
3642 3643
      if (sort_param->calc_checksum)
	param->glob_crc+= (info->checksum=
3644 3645 3646
                           (*info->s->calc_check_checksum)(info,
                                                           sort_param->
                                                           record));
unknown's avatar
unknown committed
3647 3648 3649
      DBUG_RETURN(0);
    }
  }
3650
  DBUG_RETURN(1);                               /* Impossible */
unknown's avatar
unknown committed
3651 3652 3653
}


3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667
/*
  Write record to new file.

  SYNOPSIS
    sort_write_record()
      sort_param                Sort parameters.

  NOTE
    This is only called by a master thread if parallel repair is used.

  RETURN
    0           OK
    1           Error
*/
unknown's avatar
unknown committed
3668

unknown's avatar
unknown committed
3669
int sort_write_record(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
3670 3671 3672 3673
{
  int flag;
  uint length;
  ulong block_length,reclength;
3674 3675
  uchar *from;
  uchar block_buff[8];
3676 3677
  MI_SORT_INFO *sort_info=sort_param->sort_info;
  HA_CHECK *param=sort_info->param;
unknown's avatar
unknown committed
3678 3679
  MI_INFO *info=sort_info->info;
  MYISAM_SHARE *share=info->s;
unknown's avatar
unknown committed
3680 3681
  DBUG_ENTER("sort_write_record");

unknown's avatar
unknown committed
3682
  if (sort_param->fix_datafile)
unknown's avatar
unknown committed
3683 3684
  {
    switch (sort_info->new_data_file_type) {
unknown's avatar
unknown committed
3685 3686 3687
    case BLOCK_RECORD:
      DBUG_ASSERT(0);                           /* Impossible */
      break;
unknown's avatar
unknown committed
3688
    case STATIC_RECORD:
unknown's avatar
unknown committed
3689
      if (my_b_write(&info->rec_cache,sort_param->record,
unknown's avatar
unknown committed
3690 3691 3692 3693 3694
		     share->base.pack_reclength))
      {
	mi_check_print_error(param,"%d when writing to datafile",my_errno);
	DBUG_RETURN(1);
      }
unknown's avatar
unknown committed
3695
      sort_param->filepos+=share->base.pack_reclength;
unknown's avatar
unknown committed
3696
      info->s->state.split++;
unknown's avatar
unknown committed
3697 3698 3699
      break;
    case DYNAMIC_RECORD:
      if (! info->blobs)
3700
	from=sort_param->rec_buff;
unknown's avatar
unknown committed
3701 3702 3703 3704
      else
      {
	/* must be sure that local buffer is big enough */
	reclength=info->s->base.pack_reclength+
3705
	  _mi_calc_total_blob_length(info,sort_param->record)+
unknown's avatar
unknown committed
3706 3707 3708 3709 3710
	  ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER)+MI_SPLIT_LENGTH+
	  MI_DYN_DELETE_BLOCK_HEADER;
	if (sort_info->buff_length < reclength)
	{
	  if (!(sort_info->buff=my_realloc(sort_info->buff, (uint) reclength,
3711 3712
					   MYF(MY_FREE_ON_ERROR |
					       MY_ALLOW_ZERO_PTR))))
unknown's avatar
unknown committed
3713 3714 3715
	    DBUG_RETURN(1);
	  sort_info->buff_length=reclength;
	}
3716
	from= sort_info->buff+ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER);
unknown's avatar
unknown committed
3717
      }
3718
      /* We can use info->checksum here as only one thread calls this. */
3719
      info->checksum= (*info->s->calc_check_checksum)(info,sort_param->record);
unknown's avatar
unknown committed
3720
      reclength=_mi_rec_pack(info,from,sort_param->record);
unknown's avatar
unknown committed
3721
      flag=0;
unknown's avatar
unknown committed
3722 3723

      do
unknown's avatar
unknown committed
3724
      {
unknown's avatar
unknown committed
3725 3726 3727 3728 3729 3730 3731 3732
	block_length=reclength+ 3 + test(reclength >= (65520-3));
	if (block_length < share->base.min_block_length)
	  block_length=share->base.min_block_length;
	info->update|=HA_STATE_WRITE_AT_END;
	block_length=MY_ALIGN(block_length,MI_DYN_ALIGN_SIZE);
	if (block_length > MI_MAX_BLOCK_LENGTH)
	  block_length=MI_MAX_BLOCK_LENGTH;
	if (_mi_write_part_record(info,0L,block_length,
unknown's avatar
unknown committed
3733
				  sort_param->filepos+block_length,
unknown's avatar
unknown committed
3734 3735 3736 3737 3738
				  &from,&reclength,&flag))
	{
	  mi_check_print_error(param,"%d when writing to datafile",my_errno);
	  DBUG_RETURN(1);
	}
unknown's avatar
unknown committed
3739
	sort_param->filepos+=block_length;
unknown's avatar
unknown committed
3740 3741
	info->s->state.split++;
      } while (reclength);
unknown's avatar
unknown committed
3742
      /* sort_info->param->glob_crc+=info->checksum; */
unknown's avatar
unknown committed
3743 3744 3745
      break;
    case COMPRESSED_RECORD:
      reclength=info->packed_length;
3746 3747
      length= save_pack_length((uint) share->pack.version, block_buff,
                               reclength);
unknown's avatar
unknown committed
3748
      if (info->s->base.blobs)
3749 3750
	length+= save_pack_length((uint) share->pack.version,
	                          block_buff + length, info->blob_length);
unknown's avatar
unknown committed
3751
      if (my_b_write(&info->rec_cache,block_buff,length) ||
3752
	  my_b_write(&info->rec_cache,(uchar*) sort_param->rec_buff,reclength))
unknown's avatar
unknown committed
3753 3754 3755 3756
      {
	mi_check_print_error(param,"%d when writing to datafile",my_errno);
	DBUG_RETURN(1);
      }
unknown's avatar
unknown committed
3757
      /* sort_info->param->glob_crc+=info->checksum; */
unknown's avatar
unknown committed
3758
      sort_param->filepos+=reclength+length;
unknown's avatar
unknown committed
3759
      info->s->state.split++;
unknown's avatar
unknown committed
3760 3761 3762
      break;
    }
  }
3763
  if (sort_param->master)
unknown's avatar
unknown committed
3764
  {
3765 3766 3767 3768 3769
    info->state->records++;
    if ((param->testflag & T_WRITE_LOOP) &&
        (info->state->records % WRITE_COUNT) == 0)
    {
      char llbuff[22];
3770 3771
      printf("%s\r", llstr(info->state->records,llbuff));
      VOID(fflush(stdout));
3772
    }
unknown's avatar
unknown committed
3773 3774 3775 3776 3777 3778 3779
  }
  DBUG_RETURN(0);
} /* sort_write_record */


	/* Compare two keys from _create_index_by_sort */

unknown's avatar
unknown committed
3780 3781
static int sort_key_cmp(MI_SORT_PARAM *sort_param, const void *a,
			const void *b)
unknown's avatar
unknown committed
3782
{
3783
  uint not_used[2];
3784
  return (ha_key_cmp(sort_param->seg, *((uchar**) a), *((uchar**) b),
3785
		     USE_WHOLE_KEY, SEARCH_SAME, not_used));
unknown's avatar
unknown committed
3786 3787 3788
} /* sort_key_cmp */


unknown's avatar
unknown committed
3789
static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a)
unknown's avatar
unknown committed
3790
{
3791
  uint diff_pos[2];
unknown's avatar
unknown committed
3792
  char llbuff[22],llbuff2[22];
3793 3794
  MI_SORT_INFO *sort_info=sort_param->sort_info;
  HA_CHECK *param= sort_info->param;
unknown's avatar
unknown committed
3795 3796 3797 3798
  int cmp;

  if (sort_info->key_block->inited)
  {
unknown's avatar
unknown committed
3799
    cmp=ha_key_cmp(sort_param->seg, (uchar*) sort_info->key_block->lastkey,
3800 3801
		   (uchar*) a, USE_WHOLE_KEY,
                   SEARCH_FIND | SEARCH_UPDATE | SEARCH_INSERT,
3802
		   diff_pos);
3803
    if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
unknown's avatar
unknown committed
3804
      ha_key_cmp(sort_param->seg, (uchar*) sort_info->key_block->lastkey,
3805
                 (uchar*) a, USE_WHOLE_KEY, 
3806
                 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
3807 3808
    else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
    {
3809 3810
      diff_pos[0]= mi_collect_stats_nonulls_next(sort_param->seg,
                                                 sort_param->notnull,
unknown's avatar
unknown committed
3811 3812
                                                 (uchar*) sort_info->
                                                 key_block->lastkey,
3813
                                                 (uchar*)a);
3814
    }
3815
    sort_param->unique[diff_pos[0]-1]++;
unknown's avatar
unknown committed
3816 3817 3818
  }
  else
  {
3819
    cmp= -1;
3820 3821 3822
    if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
      mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull,
                                     (uchar*)a);
unknown's avatar
unknown committed
3823
  }
unknown's avatar
unknown committed
3824
  if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0)
unknown's avatar
unknown committed
3825 3826 3827
  {
    sort_info->dupp++;
    sort_info->info->lastpos=get_record_for_key(sort_info->info,
3828
						sort_param->keyinfo,
unknown's avatar
unknown committed
3829
						(uchar*) a);
unknown's avatar
unknown committed
3830 3831 3832 3833
    mi_check_print_warning(param,
			   "Duplicate key for record at %10s against record at %10s",
			   llstr(sort_info->info->lastpos,llbuff),
			   llstr(get_record_for_key(sort_info->info,
unknown's avatar
unknown committed
3834
						    sort_param->keyinfo,
unknown's avatar
unknown committed
3835 3836
						    (uchar*) sort_info->
                                                    key_block->lastkey),
unknown's avatar
unknown committed
3837
				 llbuff2));
unknown's avatar
unknown committed
3838
    param->testflag|=T_RETRY_WITHOUT_QUICK;
unknown's avatar
unknown committed
3839
    if (sort_info->param->testflag & T_VERBOSE)
3840
      _mi_print_key(stdout,sort_param->seg,(uchar*) a, USE_WHOLE_KEY);
unknown's avatar
unknown committed
3841
    return (sort_delete_record(sort_param));
unknown's avatar
unknown committed
3842 3843 3844 3845 3846 3847 3848 3849 3850
  }
#ifndef DBUG_OFF
  if (cmp > 0)
  {
    mi_check_print_error(param,
			 "Internal error: Keys are not in order from sort");
    return(1);
  }
#endif
unknown's avatar
unknown committed
3851
  return (sort_insert_key(sort_param,sort_info->key_block,
unknown's avatar
unknown committed
3852
			  (uchar*) a, HA_OFFSET_ERROR));
unknown's avatar
unknown committed
3853 3854
} /* sort_key_write */

3855 3856
int sort_ft_buf_flush(MI_SORT_PARAM *sort_param)
{
3857
  MI_SORT_INFO *sort_info=sort_param->sort_info;
3858 3859
  SORT_KEY_BLOCKS *key_block=sort_info->key_block;
  MYISAM_SHARE *share=sort_info->info->s;
unknown's avatar
unknown committed
3860 3861
  uint val_off, val_len;
  int error;
3862 3863 3864 3865 3866
  SORT_FT_BUF *ft_buf=sort_info->ft_buf;
  uchar *from, *to;

  val_len=share->ft2_keyinfo.keylength;
  get_key_full_length_rdonly(val_off, ft_buf->lastkey);
unknown's avatar
unknown committed
3867
  to= (uchar*) ft_buf->lastkey+val_off;
3868 3869

  if (ft_buf->buf)
unknown's avatar
unknown committed
3870 3871
  {
    /* flushing first-level tree */
unknown's avatar
unknown committed
3872
    error=sort_insert_key(sort_param,key_block, (uchar*) ft_buf->lastkey,
unknown's avatar
unknown committed
3873
			  HA_OFFSET_ERROR);
3874
    for (from=to+val_len;
unknown's avatar
unknown committed
3875
         !error && from < (uchar*) ft_buf->buf;
3876 3877 3878
         from+= val_len)
    {
      memcpy(to, from, val_len);
unknown's avatar
unknown committed
3879
      error=sort_insert_key(sort_param,key_block, (uchar*) ft_buf->lastkey,
unknown's avatar
unknown committed
3880
			    HA_OFFSET_ERROR);
3881 3882 3883 3884 3885 3886 3887
    }
    return error;
  }
  /* flushing second-level tree keyblocks */
  error=flush_pending_blocks(sort_param);
  /* updating lastkey with second-level tree info */
  ft_intXstore(ft_buf->lastkey+val_off, -ft_buf->count);
unknown's avatar
unknown committed
3888 3889
  _mi_dpointer(sort_info->info, (uchar*) ft_buf->lastkey+val_off+HA_FT_WLEN,
               share->state.key_root[sort_param->key]);
3890 3891 3892 3893 3894 3895 3896
  /* restoring first level tree data in sort_info/sort_param */
  sort_info->key_block=sort_info->key_block_end- sort_info->param->sort_key_blocks;
  sort_param->keyinfo=share->keyinfo+sort_param->key;
  share->state.key_root[sort_param->key]=HA_OFFSET_ERROR;
  /* writing lastkey in first-level tree */
  return error ? error :
                 sort_insert_key(sort_param,sort_info->key_block,
unknown's avatar
unknown committed
3897
                                 (uchar*) ft_buf->lastkey,HA_OFFSET_ERROR);
3898 3899 3900 3901 3902 3903
}

static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a)
{
  uint a_len, val_off, val_len, error;
  uchar *p;
3904
  MI_SORT_INFO *sort_info=sort_param->sort_info;
3905 3906 3907
  SORT_FT_BUF *ft_buf=sort_info->ft_buf;
  SORT_KEY_BLOCKS *key_block=sort_info->key_block;

3908
  val_len= HA_FT_WLEN + sort_info->info->s->rec_reflength;
3909 3910 3911 3912 3913 3914 3915 3916 3917
  get_key_full_length_rdonly(a_len, (uchar *)a);

  if (!ft_buf)
  {
    /*
      use two-level tree only if key_reflength fits in rec_reflength place
      and row format is NOT static - for _mi_dpointer not to garble offsets
     */
    if ((sort_info->info->s->base.key_reflength <=
3918
         sort_info->info->s->rec_reflength) &&
3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933
        (sort_info->info->s->options &
          (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
      ft_buf=(SORT_FT_BUF *)my_malloc(sort_param->keyinfo->block_length +
                                      sizeof(SORT_FT_BUF), MYF(MY_WME));

    if (!ft_buf)
    {
      sort_param->key_write=sort_key_write;
      return sort_key_write(sort_param, a);
    }
    sort_info->ft_buf=ft_buf;
    goto word_init_ft_buf; /* no need to duplicate the code */
  }
  get_key_full_length_rdonly(val_off, ft_buf->lastkey);

3934
  if (ha_compare_text(sort_param->seg->charset,
3935
                      ((uchar *)a)+1,a_len-1,
unknown's avatar
unknown committed
3936
                      (uchar*) ft_buf->lastkey+1,val_off-1, 0, 0)==0)
3937 3938 3939 3940 3941
  {
    if (!ft_buf->buf) /* store in second-level tree */
    {
      ft_buf->count++;
      return sort_insert_key(sort_param,key_block,
3942
                             ((uchar *)a)+a_len, HA_OFFSET_ERROR);
3943 3944 3945
    }

    /* storing the key in the buffer. */
3946
    memcpy (ft_buf->buf, (char *)a+a_len, val_len);
3947 3948 3949 3950 3951
    ft_buf->buf+=val_len;
    if (ft_buf->buf < ft_buf->end)
      return 0;

    /* converting to two-level tree */
unknown's avatar
unknown committed
3952
    p= (uchar*) ft_buf->lastkey+val_off;
3953 3954 3955 3956 3957

    while (key_block->inited)
      key_block++;
    sort_info->key_block=key_block;
    sort_param->keyinfo=& sort_info->info->s->ft2_keyinfo;
unknown's avatar
unknown committed
3958
    ft_buf->count=((uchar*) ft_buf->buf - p)/val_len;
3959 3960

    /* flushing buffer to second-level tree */
unknown's avatar
unknown committed
3961
    for (error=0; !error && p < (uchar*) ft_buf->buf; p+= val_len)
3962 3963 3964 3965 3966
      error=sort_insert_key(sort_param,key_block,p,HA_OFFSET_ERROR);
    ft_buf->buf=0;
    return error;
  }

3967 3968 3969
  /* flushing buffer */
  if ((error=sort_ft_buf_flush(sort_param)))
    return error;
3970

3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984
word_init_ft_buf:
  a_len+=val_len;
  memcpy(ft_buf->lastkey, a, a_len);
  ft_buf->buf=ft_buf->lastkey+a_len;
  /*
    32 is just a safety margin here
    (at least max(val_len, sizeof(nod_flag)) should be there).
    May be better performance could be achieved if we'd put
      (sort_info->keyinfo->block_length-32)/XXX
      instead.
        TODO: benchmark the best value for XXX.
  */
  ft_buf->end=ft_buf->lastkey+ (sort_param->keyinfo->block_length-32);
  return 0;
3985
} /* sort_ft_key_write */
unknown's avatar
unknown committed
3986

3987

unknown's avatar
unknown committed
3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998
	/* get pointer to record from a key */

static my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo,
				   uchar *key)
{
  return _mi_dpos(info,0,key+_mi_keylength(keyinfo,key));
} /* get_record_for_key */


	/* Insert a key in sort-key-blocks */

unknown's avatar
unknown committed
3999
static int sort_insert_key(MI_SORT_PARAM *sort_param,
unknown's avatar
unknown committed
4000 4001 4002 4003
			   register SORT_KEY_BLOCKS *key_block, uchar *key,
			   my_off_t prev_block)
{
  uint a_length,t_length,nod_flag;
4004
  my_off_t filepos,key_file_length;
unknown's avatar
unknown committed
4005 4006 4007
  uchar *anc_buff,*lastkey;
  MI_KEY_PARAM s_temp;
  MI_INFO *info;
unknown's avatar
unknown committed
4008
  MI_KEYDEF *keyinfo=sort_param->keyinfo;
4009 4010
  MI_SORT_INFO *sort_info= sort_param->sort_info;
  HA_CHECK *param=sort_info->param;
unknown's avatar
unknown committed
4011 4012
  DBUG_ENTER("sort_insert_key");

unknown's avatar
unknown committed
4013
  anc_buff= (uchar*) key_block->buff;
unknown's avatar
unknown committed
4014
  info=sort_info->info;
unknown's avatar
unknown committed
4015
  lastkey= (uchar*) key_block->lastkey;
unknown's avatar
unknown committed
4016
  nod_flag= (key_block == sort_info->key_block ? 0 :
unknown's avatar
unknown committed
4017
	     info->s->base.key_reflength);
unknown's avatar
unknown committed
4018 4019 4020 4021 4022 4023 4024 4025 4026 4027

  if (!key_block->inited)
  {
    key_block->inited=1;
    if (key_block == sort_info->key_block_end)
    {
      mi_check_print_error(param,"To many key-block-levels; Try increasing sort_key_blocks");
      DBUG_RETURN(1);
    }
    a_length=2+nod_flag;
4028
    key_block->end_pos= anc_buff+2;
unknown's avatar
unknown committed
4029 4030 4031 4032 4033 4034 4035
    lastkey=0;					/* No previous key in block */
  }
  else
    a_length=mi_getint(anc_buff);

	/* Save pointer to previous block */
  if (nod_flag)
unknown's avatar
unknown committed
4036
    _mi_kpointer(info,(uchar*) key_block->end_pos,prev_block);
unknown's avatar
unknown committed
4037

unknown's avatar
unknown committed
4038 4039 4040
  t_length=(*keyinfo->pack_key)(keyinfo,nod_flag,
				(uchar*) 0,lastkey,lastkey,key,
				 &s_temp);
unknown's avatar
unknown committed
4041
  (*keyinfo->store_key)(keyinfo, (uchar*) key_block->end_pos+nod_flag,&s_temp);
unknown's avatar
unknown committed
4042 4043 4044
  a_length+=t_length;
  mi_putint(anc_buff,a_length,nod_flag);
  key_block->end_pos+=t_length;
unknown's avatar
unknown committed
4045
  if (a_length <= keyinfo->block_length)
unknown's avatar
unknown committed
4046
  {
unknown's avatar
unknown committed
4047
    VOID(_mi_move_key(keyinfo,(uchar*) key_block->lastkey,key));
unknown's avatar
unknown committed
4048 4049 4050 4051 4052 4053
    key_block->last_length=a_length-t_length;
    DBUG_RETURN(0);
  }

	/* Fill block with end-zero and write filled block */
  mi_putint(anc_buff,key_block->last_length,nod_flag);
4054
  bzero((uchar*) anc_buff+key_block->last_length,
unknown's avatar
unknown committed
4055
	keyinfo->block_length- key_block->last_length);
4056
  key_file_length=info->state->key_file_length;
4057
  if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
4058 4059 4060 4061 4062
    DBUG_RETURN(1);

  /* If we read the page from the key cache, we have to write it back to it */
  if (key_file_length == info->state->key_file_length)
  {
4063
    if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff))
4064 4065
      DBUG_RETURN(1);
  }
4066
  else if (my_pwrite(info->s->kfile,(uchar*) anc_buff,
unknown's avatar
unknown committed
4067
		     (uint) keyinfo->block_length,filepos, param->myf_rw))
unknown's avatar
unknown committed
4068
    DBUG_RETURN(1);
4069
  DBUG_DUMP("buff",(uchar*) anc_buff,mi_getint(anc_buff));
unknown's avatar
unknown committed
4070 4071

	/* Write separator-key to block in next level */
unknown's avatar
unknown committed
4072 4073
  if (sort_insert_key(sort_param,key_block+1,(uchar*) key_block->lastkey,
                      filepos))
unknown's avatar
unknown committed
4074 4075 4076 4077
    DBUG_RETURN(1);

	/* clear old block and write new key in it */
  key_block->inited=0;
unknown's avatar
unknown committed
4078
  DBUG_RETURN(sort_insert_key(sort_param, key_block,key,prev_block));
unknown's avatar
unknown committed
4079 4080 4081 4082 4083
} /* sort_insert_key */


	/* Delete record when we found a duplicated key */

unknown's avatar
unknown committed
4084
static int sort_delete_record(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
4085 4086 4087 4088
{
  uint i;
  int old_file,error;
  uchar *key;
4089 4090
  MI_SORT_INFO *sort_info=sort_param->sort_info;
  HA_CHECK *param=sort_info->param;
unknown's avatar
unknown committed
4091
  MI_INFO *info=sort_info->info;
unknown's avatar
unknown committed
4092 4093
  DBUG_ENTER("sort_delete_record");

4094
  if ((param->testflag & (T_FORCE_UNIQUENESS|T_QUICK)) == T_QUICK)
unknown's avatar
unknown committed
4095 4096
  {
    mi_check_print_error(param,
unknown's avatar
unknown committed
4097
			 "Quick-recover aborted; Run recovery without switch -q or with switch -qq");
unknown's avatar
unknown committed
4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108
    DBUG_RETURN(1);
  }
  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
  {
    mi_check_print_error(param,
			 "Recover aborted; Can't run standard recovery on compressed tables with errors in data-file. Use switch 'myisamchk --safe-recover' to fix it\n",stderr);;
    DBUG_RETURN(1);
  }

  old_file=info->dfile;
  info->dfile=info->rec_cache.file;
unknown's avatar
unknown committed
4109
  if (sort_info->current_key)
unknown's avatar
unknown committed
4110 4111
  {
    key=info->lastkey+info->s->base.max_key_length;
unknown's avatar
unknown committed
4112
    if ((error=(*info->s->read_rnd)(info,sort_param->record,info->lastpos,0)) &&
unknown's avatar
unknown committed
4113 4114 4115 4116 4117 4118 4119
	error != HA_ERR_RECORD_DELETED)
    {
      mi_check_print_error(param,"Can't read record to be removed");
      info->dfile=old_file;
      DBUG_RETURN(1);
    }

unknown's avatar
unknown committed
4120
    for (i=0 ; i < sort_info->current_key ; i++)
unknown's avatar
unknown committed
4121
    {
unknown's avatar
unknown committed
4122
      uint key_length=_mi_make_key(info,i,key,sort_param->record,info->lastpos);
unknown's avatar
unknown committed
4123 4124 4125 4126 4127 4128 4129
      if (_mi_ck_delete(info,i,key,key_length))
      {
	mi_check_print_error(param,"Can't delete key %d from record to be removed",i+1);
	info->dfile=old_file;
	DBUG_RETURN(1);
      }
    }
4130
    if (sort_param->calc_checksum)
unknown's avatar
unknown committed
4131
      param->glob_crc-=(*info->s->calc_checksum)(info, sort_param->record);
unknown's avatar
unknown committed
4132 4133 4134 4135 4136 4137 4138 4139 4140
  }
  error=flush_io_cache(&info->rec_cache) || (*info->s->delete_record)(info);
  info->dfile=old_file;				/* restore actual value */
  info->state->records--;
  DBUG_RETURN(error);
} /* sort_delete_record */

	/* Fix all pending blocks and flush everything to disk */

unknown's avatar
unknown committed
4141
int flush_pending_blocks(MI_SORT_PARAM *sort_param)
unknown's avatar
unknown committed
4142 4143
{
  uint nod_flag,length;
4144
  my_off_t filepos,key_file_length;
unknown's avatar
unknown committed
4145
  SORT_KEY_BLOCKS *key_block;
4146
  MI_SORT_INFO *sort_info= sort_param->sort_info;
4147
  myf myf_rw=sort_info->param->myf_rw;
unknown's avatar
unknown committed
4148 4149
  MI_INFO *info=sort_info->info;
  MI_KEYDEF *keyinfo=sort_param->keyinfo;
unknown's avatar
unknown committed
4150 4151 4152 4153 4154 4155 4156 4157 4158
  DBUG_ENTER("flush_pending_blocks");

  filepos= HA_OFFSET_ERROR;			/* if empty file */
  nod_flag=0;
  for (key_block=sort_info->key_block ; key_block->inited ; key_block++)
  {
    key_block->inited=0;
    length=mi_getint(key_block->buff);
    if (nod_flag)
unknown's avatar
unknown committed
4159
      _mi_kpointer(info,(uchar*) key_block->end_pos,filepos);
4160
    key_file_length=info->state->key_file_length;
4161
    bzero((uchar*) key_block->buff+length, keyinfo->block_length-length);
4162
    if ((filepos=_mi_new(info,keyinfo,DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
4163 4164 4165 4166 4167
      DBUG_RETURN(1);

    /* If we read the page from the key cache, we have to write it back */
    if (key_file_length == info->state->key_file_length)
    {
4168
      if (_mi_write_keypage(info, keyinfo, filepos,
unknown's avatar
unknown committed
4169
                            DFLT_INIT_HITS, (uchar*) key_block->buff))
4170 4171
	DBUG_RETURN(1);
    }
4172
    else if (my_pwrite(info->s->kfile,(uchar*) key_block->buff,
4173
		       (uint) keyinfo->block_length,filepos, myf_rw))
unknown's avatar
unknown committed
4174
      DBUG_RETURN(1);
4175
    DBUG_DUMP("buff",(uchar*) key_block->buff,length);
unknown's avatar
unknown committed
4176 4177
    nod_flag=1;
  }
unknown's avatar
unknown committed
4178
  info->s->state.key_root[sort_param->key]=filepos; /* Last is root for tree */
unknown's avatar
unknown committed
4179 4180 4181 4182 4183
  DBUG_RETURN(0);
} /* flush_pending_blocks */

	/* alloc space and pointers for key_blocks */

4184
static SORT_KEY_BLOCKS *alloc_key_blocks(HA_CHECK *param, uint blocks,
unknown's avatar
unknown committed
4185 4186 4187 4188 4189 4190 4191 4192 4193 4194
                                         uint buffer_length)
{
  reg1 uint i;
  SORT_KEY_BLOCKS *block;
  DBUG_ENTER("alloc_key_blocks");

  if (!(block=(SORT_KEY_BLOCKS*) my_malloc((sizeof(SORT_KEY_BLOCKS)+
					    buffer_length+IO_SIZE)*blocks,
					   MYF(0))))
  {
4195
    mi_check_print_error(param,"Not enough memory for sort-key-blocks");
unknown's avatar
unknown committed
4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212
    return(0);
  }
  for (i=0 ; i < blocks ; i++)
  {
    block[i].inited=0;
    block[i].buff=(uchar*) (block+blocks)+(buffer_length+IO_SIZE)*i;
  }
  DBUG_RETURN(block);
} /* alloc_key_blocks */


	/* Check if file is almost full */

int test_if_almost_full(MI_INFO *info)
{
  if (info->s->options & HA_OPTION_COMPRESS_RECORD)
    return 0;
4213 4214 4215 4216
  return my_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(MY_THREADSAFE)) / 10 * 9 >
         (my_off_t) info->s->base.max_key_file_length ||
         my_seek(info->dfile, 0L, MY_SEEK_END, MYF(0)) / 10 * 9 >
         (my_off_t) info->s->base.max_data_file_length;
unknown's avatar
unknown committed
4217 4218 4219 4220
}

	/* Recreate table with bigger more alloced record-data */

4221
int recreate_table(HA_CHECK *param, MI_INFO **org_info, char *filename)
unknown's avatar
unknown committed
4222 4223 4224 4225 4226
{
  int error;
  MI_INFO info;
  MYISAM_SHARE share;
  MI_KEYDEF *keyinfo,*key,*key_end;
unknown's avatar
unknown committed
4227
  HA_KEYSEG *keysegs,*keyseg;
unknown's avatar
unknown committed
4228 4229 4230 4231 4232 4233 4234
  MI_COLUMNDEF *recdef,*rec,*end;
  MI_UNIQUEDEF *uniquedef,*u_ptr,*u_end;
  MI_STATUS_INFO status_info;
  uint unpack,key_parts;
  ha_rows max_records;
  ulonglong file_length,tmp_length;
  MI_CREATE_INFO create_info;
4235
  DBUG_ENTER("recreate_table");
unknown's avatar
unknown committed
4236 4237 4238 4239 4240 4241 4242 4243 4244

  error=1;					/* Default error */
  info= **org_info;
  status_info= (*org_info)->state[0];
  info.state= &status_info;
  share= *(*org_info)->s;
  unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
    (param->testflag & T_UNPACK);
  if (!(keyinfo=(MI_KEYDEF*) my_alloca(sizeof(MI_KEYDEF)*share.base.keys)))
4245
    DBUG_RETURN(0);
4246
  memcpy((uchar*) keyinfo,(uchar*) share.keyinfo,
unknown's avatar
unknown committed
4247 4248 4249
	 (size_t) (sizeof(MI_KEYDEF)*share.base.keys));

  key_parts= share.base.all_key_parts;
unknown's avatar
unknown committed
4250
  if (!(keysegs=(HA_KEYSEG*) my_alloca(sizeof(HA_KEYSEG)*
unknown's avatar
unknown committed
4251 4252
				       (key_parts+share.base.keys))))
  {
4253
    my_afree((uchar*) keyinfo);
4254
    DBUG_RETURN(1);
unknown's avatar
unknown committed
4255 4256 4257 4258
  }
  if (!(recdef=(MI_COLUMNDEF*)
	my_alloca(sizeof(MI_COLUMNDEF)*(share.base.fields+1))))
  {
4259 4260
    my_afree((uchar*) keyinfo);
    my_afree((uchar*) keysegs);
4261
    DBUG_RETURN(1);
unknown's avatar
unknown committed
4262 4263 4264 4265
  }
  if (!(uniquedef=(MI_UNIQUEDEF*)
	my_alloca(sizeof(MI_UNIQUEDEF)*(share.state.header.uniques+1))))
  {
4266 4267 4268
    my_afree((uchar*) recdef);
    my_afree((uchar*) keyinfo);
    my_afree((uchar*) keysegs);
4269
    DBUG_RETURN(1);
unknown's avatar
unknown committed
4270 4271 4272
  }

  /* Copy the column definitions */
4273
  memcpy((uchar*) recdef,(uchar*) share.rec,
unknown's avatar
unknown committed
4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284
	 (size_t) (sizeof(MI_COLUMNDEF)*(share.base.fields+1)));
  for (rec=recdef,end=recdef+share.base.fields; rec != end ; rec++)
  {
    if (unpack && !(share.options & HA_OPTION_PACK_RECORD) &&
	rec->type != FIELD_BLOB &&
	rec->type != FIELD_VARCHAR &&
	rec->type != FIELD_CHECK)
      rec->type=(int) FIELD_NORMAL;
  }

  /* Change the new key to point at the saved key segments */
4285
  memcpy((uchar*) keysegs,(uchar*) share.keyparts,
unknown's avatar
unknown committed
4286
	 (size_t) (sizeof(HA_KEYSEG)*(key_parts+share.base.keys+
unknown's avatar
unknown committed
4287 4288 4289 4290 4291 4292 4293 4294 4295 4296
				      share.state.header.uniques)));
  keyseg=keysegs;
  for (key=keyinfo,key_end=keyinfo+share.base.keys; key != key_end ; key++)
  {
    key->seg=keyseg;
    for (; keyseg->type ; keyseg++)
    {
      if (param->language)
	keyseg->language=param->language;	/* change language */
    }
4297
    keyseg++;					/* Skip end pointer */
unknown's avatar
unknown committed
4298 4299 4300 4301
  }

  /* Copy the unique definitions and change them to point at the new key
     segments*/
4302
  memcpy((uchar*) uniquedef,(uchar*) share.uniqueinfo,
unknown's avatar
unknown committed
4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337
	 (size_t) (sizeof(MI_UNIQUEDEF)*(share.state.header.uniques)));
  for (u_ptr=uniquedef,u_end=uniquedef+share.state.header.uniques;
       u_ptr != u_end ; u_ptr++)
  {
    u_ptr->seg=keyseg;
    keyseg+=u_ptr->keysegs+1;
  }
  if (share.options & HA_OPTION_COMPRESS_RECORD)
    share.base.records=max_records=info.state->records;
  else if (share.base.min_pack_length)
    max_records=(ha_rows) (my_seek(info.dfile,0L,MY_SEEK_END,MYF(0)) /
			   (ulong) share.base.min_pack_length);
  else
    max_records=0;
  unpack= (share.options & HA_OPTION_COMPRESS_RECORD) &&
    (param->testflag & T_UNPACK);
  share.options&= ~HA_OPTION_TEMP_COMPRESS_RECORD;

  file_length=(ulonglong) my_seek(info.dfile,0L,MY_SEEK_END,MYF(0));
  tmp_length= file_length+file_length/10;
  set_if_bigger(file_length,param->max_data_file_length);
  set_if_bigger(file_length,tmp_length);
  set_if_bigger(file_length,(ulonglong) share.base.max_data_file_length);

  VOID(mi_close(*org_info));
  bzero((char*) &create_info,sizeof(create_info));
  create_info.max_rows=max(max_records,share.base.records);
  create_info.reloc_rows=share.base.reloc;
  create_info.old_options=(share.options |
			   (unpack ? HA_OPTION_TEMP_COMPRESS_RECORD : 0));

  create_info.data_file_length=file_length;
  create_info.auto_increment=share.state.auto_increment;
  create_info.language = (param->language ? param->language :
			  share.state.header.language);
unknown's avatar
unknown committed
4338
  create_info.key_file_length=  status_info.key_file_length;
4339 4340 4341 4342 4343
  /*
    Allow for creating an auto_increment key. This has an effect only if
    an auto_increment key exists in the original table.
  */
  create_info.with_auto_increment= TRUE;
unknown's avatar
unknown committed
4344 4345 4346
  /* We don't have to handle symlinks here because we are using
     HA_DONT_TOUCH_DATA */
  if (mi_create(filename,
unknown's avatar
unknown committed
4347 4348 4349 4350 4351 4352 4353 4354 4355
		share.base.keys - share.state.header.uniques,
		keyinfo, share.base.fields, recdef,
		share.state.header.uniques, uniquedef,
		&create_info,
		HA_DONT_TOUCH_DATA))
  {
    mi_check_print_error(param,"Got error %d when trying to recreate indexfile",my_errno);
    goto end;
  }
unknown's avatar
unknown committed
4356
  *org_info=mi_open(filename,O_RDWR,
unknown's avatar
unknown committed
4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373
		    (param->testflag & T_WAIT_FOREVER) ? HA_OPEN_WAIT_IF_LOCKED :
		    (param->testflag & T_DESCRIPT) ? HA_OPEN_IGNORE_IF_LOCKED :
		    HA_OPEN_ABORT_IF_LOCKED);
  if (!*org_info)
  {
    mi_check_print_error(param,"Got error %d when trying to open re-created indexfile",
		my_errno);
    goto end;
  }
  /* We are modifing */
  (*org_info)->s->options&= ~HA_OPTION_READ_ONLY_DATA;
  VOID(_mi_readinfo(*org_info,F_WRLCK,0));
  (*org_info)->state->records=info.state->records;
  if (share.state.create_time)
    (*org_info)->s->state.create_time=share.state.create_time;
  (*org_info)->s->state.unique=(*org_info)->this_unique=
    share.state.unique;
4374
  (*org_info)->state->checksum=info.state->checksum;
unknown's avatar
unknown committed
4375 4376 4377 4378 4379 4380 4381 4382 4383
  (*org_info)->state->del=info.state->del;
  (*org_info)->s->state.dellink=share.state.dellink;
  (*org_info)->state->empty=info.state->empty;
  (*org_info)->state->data_file_length=info.state->data_file_length;
  if (update_state_info(param,*org_info,UPDATE_TIME | UPDATE_STAT |
			UPDATE_OPEN_COUNT))
    goto end;
  error=0;
end:
4384 4385 4386 4387
  my_afree((uchar*) uniquedef);
  my_afree((uchar*) keyinfo);
  my_afree((uchar*) recdef);
  my_afree((uchar*) keysegs);
4388
  DBUG_RETURN(error);
unknown's avatar
unknown committed
4389 4390 4391 4392 4393
}


	/* write suffix to data file if neaded */

4394
int write_data_suffix(MI_SORT_INFO *sort_info, my_bool fix_datafile)
unknown's avatar
unknown committed
4395
{
unknown's avatar
unknown committed
4396 4397 4398
  MI_INFO *info=sort_info->info;

  if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile)
unknown's avatar
unknown committed
4399
  {
4400
    uchar buff[MEMMAP_EXTRA_MARGIN];
unknown's avatar
unknown committed
4401 4402 4403
    bzero(buff,sizeof(buff));
    if (my_b_write(&info->rec_cache,buff,sizeof(buff)))
    {
unknown's avatar
unknown committed
4404
      mi_check_print_error(sort_info->param,
unknown's avatar
unknown committed
4405
			   "%d when writing to datafile",my_errno);
unknown's avatar
unknown committed
4406 4407
      return 1;
    }
unknown's avatar
unknown committed
4408
    sort_info->param->read_cache.end_of_file+=sizeof(buff);
unknown's avatar
unknown committed
4409 4410 4411 4412 4413 4414
  }
  return 0;
}

	/* Update state and myisamchk_time of indexfile */

4415
int update_state_info(HA_CHECK *param, MI_INFO *info,uint update)
unknown's avatar
unknown committed
4416 4417 4418 4419 4420 4421 4422 4423 4424 4425
{
  MYISAM_SHARE *share=info->s;

  if (update & UPDATE_OPEN_COUNT)
  {
    share->state.open_count=0;
    share->global_changed=0;
  }
  if (update & UPDATE_STAT)
  {
4426
    uint i, key_parts= mi_uint2korr(share->state.header.key_parts);
unknown's avatar
unknown committed
4427
    share->state.rec_per_key_rows=info->state->records;
4428
    share->state.changed&= ~STATE_NOT_ANALYZED;
unknown's avatar
unknown committed
4429
    if (info->state->records)
4430
    {
unknown's avatar
unknown committed
4431 4432 4433 4434 4435
      for (i=0; i<key_parts; i++)
      {
        if (!(share->state.rec_per_key_part[i]=param->rec_per_key_part[i]))
          share->state.changed|= STATE_NOT_ANALYZED;
      }
4436
    }
unknown's avatar
unknown committed
4437 4438 4439 4440 4441
  }
  if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC))
  {
    if (update & UPDATE_TIME)
    {
4442
      share->state.check_time= time((time_t*) 0);
unknown's avatar
unknown committed
4443 4444 4445
      if (!share->state.create_time)
	share->state.create_time=share->state.check_time;
    }
unknown's avatar
unknown committed
4446 4447 4448
    /*
      When tables are locked we haven't synched the share state and the
      real state for a while so we better do it here before synching
unknown's avatar
unknown committed
4449 4450
      the share state to disk. Only when table is write locked is it
      necessary to perform this synch.
unknown's avatar
unknown committed
4451
    */
unknown's avatar
unknown committed
4452 4453
    if (info->lock_type == F_WRLCK)
      share->state.state= *info->state;
unknown's avatar
unknown committed
4454 4455
    if (mi_state_info_write(share->kfile,&share->state,1+2))
      goto err;
unknown's avatar
unknown committed
4456
    share->changed=0;
unknown's avatar
unknown committed
4457 4458 4459 4460
  }
  {						/* Force update of status */
    int error;
    uint r_locks=share->r_locks,w_locks=share->w_locks;
4461
    share->r_locks= share->w_locks= share->tot_locks= 0;
unknown's avatar
unknown committed
4462
    error=_mi_writeinfo(info,WRITEINFO_NO_UNLOCK);
4463 4464 4465
    share->r_locks=r_locks;
    share->w_locks=w_locks;
    share->tot_locks=r_locks+w_locks;
unknown's avatar
unknown committed
4466 4467 4468 4469
    if (!error)
      return 0;
  }
err:
unknown's avatar
unknown committed
4470
  mi_check_print_error(param,"%d when updating keyfile",my_errno);
unknown's avatar
unknown committed
4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486
  return 1;
}

	/*
	  Update auto increment value for a table
	  When setting the 'repair_only' flag we only want to change the
	  old auto_increment value if its wrong (smaller than some given key).
	  The reason is that we shouldn't change the auto_increment value
	  for a table without good reason when only doing a repair; If the
	  user have inserted and deleted rows, the auto_increment value
	  may be bigger than the biggest current row and this is ok.

	  If repair_only is not set, we will update the flag to the value in
	  param->auto_increment is bigger than the biggest key.
	*/

4487
void update_auto_increment_key(HA_CHECK *param, MI_INFO *info,
unknown's avatar
unknown committed
4488 4489
			       my_bool repair_only)
{
4490
  uchar *record= 0;
4491 4492
  DBUG_ENTER("update_auto_increment_key");

unknown's avatar
unknown committed
4493
  if (!info->s->base.auto_key ||
4494
      ! mi_is_key_active(info->s->state.key_map, info->s->base.auto_key - 1))
unknown's avatar
unknown committed
4495
  {
unknown's avatar
unknown committed
4496 4497 4498 4499
    if (!(param->testflag & T_VERY_SILENT))
      mi_check_print_info(param,
			  "Table: %s doesn't have an auto increment key\n",
			  param->isam_file_name);
4500
    DBUG_VOID_RETURN;
unknown's avatar
unknown committed
4501 4502
  }
  if (!(param->testflag & T_SILENT) &&
4503
      !(param->testflag & T_REP))
unknown's avatar
unknown committed
4504
    printf("Updating MyISAM file: %s\n", param->isam_file_name);
4505 4506 4507 4508
  /*
    We have to use an allocated buffer instead of info->rec_buff as 
    _mi_put_key_in_record() may use info->rec_buff
  */
4509
  if (!mi_alloc_rec_buff(info, -1, &record))
4510 4511
  {
    mi_check_print_error(param,"Not enough memory for extra record");
4512
    DBUG_VOID_RETURN;
4513 4514
  }

unknown's avatar
unknown committed
4515
  mi_extra(info,HA_EXTRA_KEYREAD,0);
4516
  if (mi_rlast(info, record, info->s->base.auto_key-1))
unknown's avatar
unknown committed
4517 4518 4519
  {
    if (my_errno != HA_ERR_END_OF_FILE)
    {
unknown's avatar
unknown committed
4520
      mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4521
      my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
unknown's avatar
unknown committed
4522
      mi_check_print_error(param,"%d when reading last record",my_errno);
4523
      DBUG_VOID_RETURN;
unknown's avatar
unknown committed
4524 4525 4526 4527 4528 4529
    }
    if (!repair_only)
      info->s->state.auto_increment=param->auto_increment_value;
  }
  else
  {
4530
    ulonglong auto_increment= retrieve_auto_increment(info, record);
unknown's avatar
unknown committed
4531
    set_if_bigger(info->s->state.auto_increment,auto_increment);
4532 4533
    if (!repair_only)
      set_if_bigger(info->s->state.auto_increment, param->auto_increment_value);
unknown's avatar
unknown committed
4534
  }
unknown's avatar
unknown committed
4535
  mi_extra(info,HA_EXTRA_NO_KEYREAD,0);
4536
  my_free(mi_get_rec_buff_ptr(info, record), MYF(0));
unknown's avatar
unknown committed
4537
  update_state_info(param, info, UPDATE_AUTO_INC);
4538
  DBUG_VOID_RETURN;
unknown's avatar
unknown committed
4539 4540
}

4541 4542 4543

/*
  Update statistics for each part of an index
4544

4545 4546
  SYNOPSIS
    update_key_parts()
4547
      keyinfo           IN  Index information (only key->keysegs used)
4548
      rec_per_key_part  OUT Store statistics here
4549 4550
      unique            IN  Array of (#distinct tuples)
      notnull_tuples    IN  Array of (#tuples), or NULL
4551
      records               Number of records in the table
4552 4553 4554 4555 4556 4557 4558

  DESCRIPTION
    This function is called produce index statistics values from unique and 
    notnull_tuples arrays after these arrays were produced with sequential
    index scan (the scan is done in two places: chk_index() and
    sort_key_write()).

4559 4560
    This function handles all 3 index statistics collection methods.

4561
    Unique is an array:
4562 4563 4564 4565 4566 4567 4568 4569 4570 4571
      unique[0]= (#different values of {keypart1}) - 1
      unique[1]= (#different values of {keypart1,keypart2} tuple)-unique[0]-1
      ...

    For MI_STATS_METHOD_IGNORE_NULLS method, notnull_tuples is an array too:
      notnull_tuples[0]= (#of {keypart1} tuples such that keypart1 is not NULL)
      notnull_tuples[1]= (#of {keypart1,keypart2} tuples such that all 
                          keypart{i} are not NULL)
      ...
    For all other statistics collection methods notnull_tuples==NULL.
4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582

    Output is an array:
    rec_per_key_part[k] = 
     = E(#records in the table such that keypart_1=c_1 AND ... AND 
         keypart_k=c_k for arbitrary constants c_1 ... c_k) 
     
     = {assuming that values have uniform distribution and index contains all
        tuples from the domain (or that {c_1, ..., c_k} tuple is choosen from
        index tuples}
     
     = #tuples-in-the-index / #distinct-tuples-in-the-index.
4583 4584 4585 4586 4587 4588 4589 4590
    
    The #tuples-in-the-index and #distinct-tuples-in-the-index have different 
    meaning depending on which statistics collection method is used:
    
    MI_STATS_METHOD_*  how are nulls compared?  which tuples are counted?
     NULLS_EQUAL            NULL == NULL           all tuples in table
     NULLS_NOT_EQUAL        NULL != NULL           all tuples in table
     IGNORE_NULLS               n/a             tuples that don't have NULLs
4591
*/
unknown's avatar
unknown committed
4592

unknown's avatar
unknown committed
4593
void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part,
4594 4595
                      ulonglong *unique, ulonglong *notnull,
                      ulonglong records)
unknown's avatar
unknown committed
4596
{
4597 4598
  ulonglong count=0,tmp, unique_tuples;
  ulonglong tuples= records;
unknown's avatar
unknown committed
4599 4600 4601 4602
  uint parts;
  for (parts=0 ; parts < keyinfo->keysegs  ; parts++)
  {
    count+=unique[parts];
4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618
    unique_tuples= count + 1;    
    if (notnull)
    {
      tuples= notnull[parts];
      /* 
        #(unique_tuples not counting tuples with NULLs) = 
          #(unique_tuples counting tuples with NULLs as different) - 
          #(tuples with NULLs)
      */
      unique_tuples -= (records - notnull[parts]);
    }
    
    if (unique_tuples == 0)
      tmp= 1;
    else if (count == 0)
      tmp= tuples; /* 1 unique tuple */
unknown's avatar
unknown committed
4619
    else
4620 4621 4622 4623 4624 4625
      tmp= (tuples + unique_tuples/2) / unique_tuples;

    /* 
      for some weird keys (e.g. FULLTEXT) tmp can be <1 here. 
      let's ensure it is not
    */
4626
    set_if_bigger(tmp,1);
4627 4628 4629
    /* Keys are stored as 32 byte int's; Ensure we don't get an overflow */
    if (tmp >= (ulonglong) ~(uint32) 0)
      tmp=(ulonglong) ~(uint32) 0;
4630

unknown's avatar
unknown committed
4631 4632 4633 4634 4635 4636
    *rec_per_key_part=(ulong) tmp;
    rec_per_key_part++;
  }
}


4637
static ha_checksum mi_byte_checksum(const uchar *buf, uint length)
unknown's avatar
unknown committed
4638 4639
{
  ha_checksum crc;
4640
  const uchar *end=buf+length;
unknown's avatar
unknown committed
4641 4642 4643 4644 4645
  for (crc=0; buf != end; buf++)
    crc=((crc << 1) + *((uchar*) buf)) +
      test(crc & (((ha_checksum) 1) << (8*sizeof(ha_checksum)-1)));
  return crc;
}
4646 4647 4648

static my_bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows)
{
4649 4650
  uint key_maxlength=key->maxlength;
  if (key->flag & HA_FULLTEXT)
4651 4652 4653 4654 4655
  {
    uint ft_max_word_len_for_sort=FT_MAX_WORD_LEN_FOR_SORT*
                                  key->seg->charset->mbmaxlen;
    key_maxlength+=ft_max_word_len_for_sort-HA_FT_MAXBYTELEN;
  }
4656 4657
  return (key->flag & HA_SPATIAL) ||
          (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) &&
4658
	  ((ulonglong) rows * key_maxlength >
4659
	   (ulonglong) myisam_max_temp_length));
4660 4661
}

4662 4663 4664 4665 4666 4667 4668 4669
/*
  Deactivate all not unique index that can be recreated fast
  These include packed keys on which sorting will use more temporary
  space than the max allowed file length or for which the unpacked keys
  will take much more space than packed keys.
  Note that 'rows' may be zero for the case when we don't know how many
  rows we will put into the file.
 */
4670

4671
void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows)
4672 4673
{
  MYISAM_SHARE *share=info->s;
4674 4675 4676 4677 4678 4679
  MI_KEYDEF    *key=share->keyinfo;
  uint          i;

  DBUG_ASSERT(info->state->records == 0 &&
              (!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES));
  for (i=0 ; i < share->base.keys ; i++,key++)
4680
  {
4681 4682
    if (!(key->flag & (HA_NOSAME | HA_SPATIAL | HA_AUTO_KEY)) &&
        ! mi_too_big_key_for_sort(key,rows) && info->s->base.auto_key != i+1)
4683
    {
4684
      mi_clear_key_active(share->state.key_map, i);
4685
      info->update|= HA_STATE_CHANGED;
4686 4687 4688 4689 4690
    }
  }
}


4691 4692 4693 4694 4695
/*
  Return TRUE if we can use repair by sorting
  One can set the force argument to force to use sorting
  even if the temporary file would be quite big!
*/
4696

unknown's avatar
unknown committed
4697
my_bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows,
unknown's avatar
unknown committed
4698
			    ulonglong key_map, my_bool force)
4699 4700 4701
{
  MYISAM_SHARE *share=info->s;
  MI_KEYDEF *key=share->keyinfo;
4702 4703 4704
  uint i;

  /*
unknown's avatar
unknown committed
4705
    mi_repair_by_sort only works if we have at least one key. If we don't
4706 4707
    have any keys, we should use the normal repair.
  */
4708
  if (! mi_is_any_key_active(key_map))
4709 4710 4711
    return FALSE;				/* Can't use sort */
  for (i=0 ; i < share->base.keys ; i++,key++)
  {
4712
    if (!force && mi_too_big_key_for_sort(key,rows))
4713 4714 4715 4716
      return FALSE;
  }
  return TRUE;
}
4717 4718 4719


static void
4720
set_data_file_type(MI_SORT_INFO *sort_info, MYISAM_SHARE *share)
4721 4722
{
  if ((sort_info->new_data_file_type=share->data_file_type) ==
unknown's avatar
unknown committed
4723
      COMPRESSED_RECORD && sort_info->param->testflag & T_UNPACK)
4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738
  {
    MYISAM_SHARE tmp;

    if (share->options & HA_OPTION_PACK_RECORD)
      sort_info->new_data_file_type = DYNAMIC_RECORD;
    else
      sort_info->new_data_file_type = STATIC_RECORD;

    /* Set delete_function for sort_delete_record() */
    memcpy((char*) &tmp, share, sizeof(*share));
    tmp.options= ~HA_OPTION_COMPRESS_RECORD;
    mi_setup_functions(&tmp);
    share->delete_record=tmp.delete_record;
  }
}
4739 4740 4741 4742 4743 4744 4745

int mi_make_backup_of_index(MI_INFO *info, time_t backup_time, myf flags)
{
  char backup_name[FN_REFLEN + MY_BACKUP_NAME_EXTRA_LENGTH];
  my_create_backup_name(backup_name, info->s->index_file_name, backup_time);
  return my_copy(info->s->index_file_name, backup_name, flags);
}