mysqlhotcopy.sh 27.5 KB
Newer Older
unknown's avatar
unknown committed
1
#!@PERL@ -w
unknown's avatar
unknown committed
2 3 4 5 6 7 8

use strict;
use Getopt::Long;
use Data::Dumper;
use File::Basename;
use File::Path;
use DBI;
9
use Sys::Hostname;
unknown's avatar
unknown committed
10 11 12

=head1 NAME

unknown's avatar
unknown committed
13
mysqlhotcopy - fast on-line hot-backup utility for local MySQL databases and tables
unknown's avatar
unknown committed
14 15 16 17 18 19 20 21 22

=head1 SYNOPSIS

  mysqlhotcopy db_name

  mysqlhotcopy --suffix=_copy db_name_1 ... db_name_n

  mysqlhotcopy db_name_1 ... db_name_n /path/to/new_directory

unknown's avatar
unknown committed
23 24 25 26 27 28 29 30 31 32 33
  mysqlhotcopy db_name./regex/

  mysqlhotcopy db_name./^\(foo\|bar\)/

  mysqlhotcopy db_name./~regex/

  mysqlhotcopy db_name_1./regex_1/ db_name_1./regex_2/ ... db_name_n./regex_n/ /path/to/new_directory

  mysqlhotcopy --method='scp -Bq -i /usr/home/foo/.ssh/identity' --user=root --password=secretpassword \
         db_1./^nice_table/ user@some.system.dom:~/path/to/new_directory

unknown's avatar
unknown committed
34
WARNING: THIS PROGRAM IS STILL IN BETA. Comments/patches welcome.
unknown's avatar
unknown committed
35 36 37 38 39

=cut

# Documentation continued at end of file

40
my $VERSION = "1.17";
unknown's avatar
unknown committed
41 42

my $opt_tmpdir = $ENV{TMPDIR} || "/tmp";
unknown's avatar
unknown committed
43 44 45

my $OPTIONS = <<"_OPTIONS";

unknown's avatar
unknown committed
46 47
$0 Ver $VERSION

unknown's avatar
unknown committed
48
Usage: $0 db_name[./table_regex/] [new_db_name | directory]
unknown's avatar
unknown committed
49 50 51 52

  -?, --help           display this helpscreen and exit
  -u, --user=#         user for database login if not current user
  -p, --password=#     password to use when connecting to server
53 54
  -h, --host=#	       Hostname for local server when connecting over TCP/IP
  -P, --port=#         port to use when connecting to local server with TCP/IP
unknown's avatar
unknown committed
55 56
  -S, --socket=#       socket to use when connecting to local server

unknown's avatar
unknown committed
57 58 59
  --allowold           don\'t abort if target already exists (rename it _old)
  --keepold            don\'t delete previous (now renamed) target when done
  --noindices          don\'t include full index files in copy
unknown's avatar
unknown committed
60 61 62 63 64 65 66 67 68 69
  --method=#           method for copy (only "cp" currently supported)

  -q, --quiet          be silent except for errors
  --debug              enable debug
  -n, --dryrun         report actions without doing them

  --regexp=#           copy all databases with names matching regexp
  --suffix=#           suffix for names of copied databases
  --checkpoint=#       insert checkpoint entry into specified db.table
  --flushlog           flush logs once all tables are locked 
unknown's avatar
unknown committed
70 71
  --resetmaster        reset the binlog once all tables are locked
  --resetslave         reset the master.info once all tables are locked
unknown's avatar
unknown committed
72
  --tmpdir=#	       temporary directory (instead of $opt_tmpdir)
73
  --record_log_pos=#   record slave and master status in specified db.table
unknown's avatar
unknown committed
74

unknown's avatar
unknown committed
75
  Try \'perldoc $0 for more complete documentation\'
unknown's avatar
unknown committed
76 77 78 79 80 81 82
_OPTIONS

sub usage {
    die @_, $OPTIONS;
}

my %opt = (
unknown's avatar
unknown committed
83
    user	=> scalar getpwuid($>),
unknown's avatar
unknown committed
84
    noindices	=> 0,
unknown's avatar
unknown committed
85 86 87 88 89 90 91 92
    allowold	=> 0,	# for safety
    keepold	=> 0,
    method	=> "cp",
    flushlog    => 0,
);
Getopt::Long::Configure(qw(no_ignore_case)); # disambuguate -p and -P
GetOptions( \%opt,
    "help",
unknown's avatar
unknown committed
93
    "host|h=s",
unknown's avatar
unknown committed
94 95 96 97 98 99
    "user|u=s",
    "password|p=s",
    "port|P=s",
    "socket|S=s",
    "allowold!",
    "keepold!",
unknown's avatar
unknown committed
100
    "noindices!",
unknown's avatar
unknown committed
101 102 103 104 105 106 107
    "method=s",
    "debug",
    "quiet|q",
    "mv!",
    "regexp=s",
    "suffix=s",
    "checkpoint=s",
108
    "record_log_pos=s",
unknown's avatar
unknown committed
109
    "flushlog",
unknown's avatar
unknown committed
110 111
    "resetmaster",
    "resetslave",
unknown's avatar
unknown committed
112
    "tmpdir|t=s",
unknown's avatar
unknown committed
113 114 115 116 117 118 119
    "dryrun|n",
) or usage("Invalid option");

# @db_desc
# ==========
# a list of hash-refs containing:
#
unknown's avatar
unknown committed
120 121 122 123 124
#   'src'     - name of the db to copy
#   't_regex' - regex describing tables in src
#   'target'  - destination directory of the copy
#   'tables'  - array-ref to list of tables in the db
#   'files'   - array-ref to list of files to be copied
unknown's avatar
unknown committed
125
#               (RAID files look like 'nn/name.MYD')
126
#   'index'   - array-ref to list of indexes to be copied
unknown's avatar
unknown committed
127 128 129 130
#

my @db_desc = ();
my $tgt_name = undef;
unknown's avatar
unknown committed
131

unknown's avatar
unknown committed
132 133
usage("") if ($opt{help});

unknown's avatar
unknown committed
134 135
if ( $opt{regexp} || $opt{suffix} || @ARGV > 2 ) {
    $tgt_name   = pop @ARGV unless ( exists $opt{suffix} );
unknown's avatar
unknown committed
136
    @db_desc = map { s{^([^\.]+)\./(.+)/$}{$1}; { 'src' => $_, 't_regex' => ( $2 ? $2 : '.*' ) } } @ARGV;
unknown's avatar
unknown committed
137 138 139 140
}
else {
    usage("Database name to hotcopy not specified") unless ( @ARGV );

unknown's avatar
unknown committed
141 142 143
    $ARGV[0] =~ s{^([^\.]+)\./(.+)/$}{$1};
    @db_desc = ( { 'src' => $ARGV[0], 't_regex' => ( $2 ? $2 : '.*' ) } );

unknown's avatar
unknown committed
144 145 146 147 148 149 150 151 152 153
    if ( @ARGV == 2 ) {
	$tgt_name   = $ARGV[1];
    }
    else {
	$opt{suffix} = "_copy";
    }
}

my %mysqld_vars;
my $start_time = time;
unknown's avatar
unknown committed
154
$opt_tmpdir= $opt{tmpdir} if $opt{tmpdir};
unknown's avatar
unknown committed
155 156 157 158 159
$0 = $1 if $0 =~ m:/([^/]+)$:;
$opt{quiet} = 0 if $opt{debug};
$opt{allowold} = 1 if $opt{keepold};

# --- connect to the database ---
160 161
my $dsn;
$dsn  = ";host=" . (defined($opt{host}) ? $opt{host} : "localhost");
unknown's avatar
unknown committed
162 163 164
$dsn .= ";port=$opt{port}" if $opt{port};
$dsn .= ";mysql_socket=$opt{socket}" if $opt{socket};

unknown's avatar
unknown committed
165 166 167
my $dbh = DBI->connect("dbi:mysql:$dsn;mysql_read_default_group=mysqlhotcopy",
                        $opt{user}, $opt{password},
{
unknown's avatar
unknown committed
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
    RaiseError => 1,
    PrintError => 0,
    AutoCommit => 1,
});

# --- check that checkpoint table exists if specified ---
if ( $opt{checkpoint} ) {
    eval { $dbh->do( qq{ select time_stamp, src, dest, msg 
			 from $opt{checkpoint} where 1 != 1} );
       };

    die "Error accessing Checkpoint table ($opt{checkpoint}): $@"
      if ( $@ );
}

183 184 185 186 187 188 189 190 191 192
# --- check that log_pos table exists if specified ---
if ( $opt{record_log_pos} ) {
    eval { $dbh->do( qq{ select host, time_stamp, log_file, log_pos, master_host, master_log_file, master_log_pos
			 from $opt{record_log_pos} where 1 != 1} );
       };

    die "Error accessing log_pos table ($opt{record_log_pos}): $@"
      if ( $@ );
}

unknown's avatar
unknown committed
193
# --- get variables from database ---
unknown's avatar
unknown committed
194
my $sth_vars = $dbh->prepare("show variables like 'datadir'");
unknown's avatar
unknown committed
195 196 197 198
$sth_vars->execute;
while ( my ($var,$value) = $sth_vars->fetchrow_array ) {
    $mysqld_vars{ $var } = $value;
}
unknown's avatar
unknown committed
199
my $datadir = $mysqld_vars{'datadir'}
unknown's avatar
unknown committed
200 201 202 203 204
    || die "datadir not in mysqld variables";
$datadir =~ s:/$::;


# --- get target path ---
unknown's avatar
unknown committed
205 206
my ($tgt_dirname, $to_other_database);
$to_other_database=0;
unknown's avatar
unknown committed
207
if (defined($tgt_name) && $tgt_name =~ m:^\w+$: && @db_desc <= 1)
unknown's avatar
unknown committed
208
{
unknown's avatar
unknown committed
209
    $tgt_dirname = "$datadir/$tgt_name";
unknown's avatar
unknown committed
210
    $to_other_database=1;
unknown's avatar
unknown committed
211
}
unknown's avatar
unknown committed
212
elsif (defined($tgt_name) && ($tgt_name =~ m:/: || $tgt_name eq '.')) {
unknown's avatar
unknown committed
213 214 215
    $tgt_dirname = $tgt_name;
}
elsif ( $opt{suffix} ) {
unknown's avatar
unknown committed
216
    print "Using copy suffix '$opt{suffix}'\n" unless $opt{quiet};
unknown's avatar
unknown committed
217
}
unknown's avatar
unknown committed
218 219 220 221
else
{
  $tgt_name="" if (!defined($tgt_name));
  die "Target '$tgt_name' doesn't look like a database name or directory path.\n";
unknown's avatar
unknown committed
222 223 224 225 226 227 228 229 230 231 232 233 234 235
}

# --- resolve database names from regexp ---
if ( defined $opt{regexp} ) {
    my $sth_dbs = $dbh->prepare("show databases");
    $sth_dbs->execute;
    while ( my ($db_name) = $sth_dbs->fetchrow_array ) {
	push @db_desc, { 'src' => $db_name } if ( $db_name =~ m/$opt{regexp}/o );
    }
}

# --- get list of tables to hotcopy ---

my $hc_locks = "";
236
my $hc_tables = "";
unknown's avatar
unknown committed
237 238 239 240 241
my $num_tables = 0;
my $num_files = 0;

foreach my $rdb ( @db_desc ) {
    my $db = $rdb->{src};
unknown's avatar
unknown committed
242
    my @dbh_tables = get_list_of_tables( $db );
unknown's avatar
unknown committed
243

unknown's avatar
unknown committed
244
    ## generate regex for tables/files
unknown's avatar
unknown committed
245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
    my $t_regex;
    my $negated;
    if ($rdb->{t_regex}) {
        $t_regex = $rdb->{t_regex};        ## assign temporary regex
        $negated = $t_regex =~ tr/~//d;    ## remove and count
                                           ## negation operator: we
                                           ## don't allow ~ in table
                                           ## names

        $t_regex = qr/$t_regex/;           ## make regex string from
                                           ## user regex

        ## filter (out) tables specified in t_regex
        print "Filtering tables with '$t_regex'\n" if $opt{debug};
        @dbh_tables = ( $negated 
                        ? grep { $_ !~ $t_regex } @dbh_tables
                        : grep { $_ =~ $t_regex } @dbh_tables );
    }
unknown's avatar
unknown committed
263 264

    ## get list of files to copy
unknown's avatar
unknown committed
265 266 267 268
    my $db_dir = "$datadir/$db";
    opendir(DBDIR, $db_dir ) 
      or die "Cannot open dir '$db_dir': $!";

unknown's avatar
unknown committed
269
    my %db_files;
unknown's avatar
unknown committed
270 271 272 273 274 275 276 277 278 279 280 281 282 283
    my @raid_dir = ();

    while ( defined( my $name = readdir DBDIR ) ) {
	if ( $name =~ /^\d\d$/ && -d "$db_dir/$name" ) {
	    push @raid_dir, $name;
	}
	else {
	    $db_files{$name} = $1 if ( $name =~ /(.+)\.\w+$/ );
        }
    }
    closedir( DBDIR );

    scan_raid_dir( \%db_files, $db_dir, @raid_dir );

unknown's avatar
unknown committed
284 285 286
    unless( keys %db_files ) {
	warn "'$db' is an empty database\n";
    }
unknown's avatar
unknown committed
287

unknown's avatar
unknown committed
288
    ## filter (out) files specified in t_regex
unknown's avatar
unknown committed
289 290 291 292 293 294 295 296 297 298
    my @db_files;
    if ($rdb->{t_regex}) {
        @db_files = ($negated
                     ? grep { $db_files{$_} !~ $t_regex } keys %db_files
                     : grep { $db_files{$_} =~ $t_regex } keys %db_files );
    }
    else {
        @db_files = keys %db_files;
    }

unknown's avatar
unknown committed
299
    @db_files = sort @db_files;
unknown's avatar
unknown committed
300

unknown's avatar
unknown committed
301
    my @index_files=();
unknown's avatar
unknown committed
302 303

    ## remove indices unless we're told to keep them
unknown's avatar
unknown committed
304 305
    if ($opt{noindices}) {
        @index_files= grep { /\.(ISM|MYI)$/ } @db_files;
unknown's avatar
unknown committed
306 307 308 309
	@db_files = grep { not /\.(ISM|MYI)$/ } @db_files;
    }

    $rdb->{files}  = [ @db_files ];
unknown's avatar
unknown committed
310
    $rdb->{index}  = [ @index_files ];
unknown's avatar
unknown committed
311
    my @hc_tables = map { "`$db`.`$_`" } @dbh_tables;
unknown's avatar
unknown committed
312 313
    $rdb->{tables} = [ @hc_tables ];

unknown's avatar
unknown committed
314 315
    $rdb->{raid_dirs} = [ get_raid_dirs( $rdb->{files} ) ];

unknown's avatar
unknown committed
316 317
    $hc_locks .= ", "  if ( length $hc_locks && @hc_tables );
    $hc_locks .= join ", ", map { "$_ READ" } @hc_tables;
unknown's avatar
unknown committed
318
    $hc_tables .= ", "  if ( length $hc_tables && @hc_tables );
319
    $hc_tables .= join ", ", @hc_tables;
unknown's avatar
unknown committed
320 321 322 323 324 325 326

    $num_tables += scalar @hc_tables;
    $num_files  += scalar @{$rdb->{files}};
}

# --- resolve targets for copies ---

unknown's avatar
unknown committed
327
if (defined($tgt_name) && length $tgt_name ) {
unknown's avatar
unknown committed
328 329 330
    # explicit destination directory specified

    # GNU `cp -r` error message
unknown's avatar
unknown committed
331 332 333 334 335 336 337 338
    die "copying multiple databases, but last argument ($tgt_dirname) is not a directory\n"
      if ( @db_desc > 1 && !(-e $tgt_dirname && -d $tgt_dirname ) );

    if ($to_other_database)
    {
      foreach my $rdb ( @db_desc ) {
	$rdb->{target} = "$tgt_dirname";
      }
unknown's avatar
unknown committed
339
    }
unknown's avatar
unknown committed
340 341 342 343 344 345
    elsif ($opt{method} =~ /^scp\b/) 
    {   # we have to trust scp to hit the target
	foreach my $rdb ( @db_desc ) {
	    $rdb->{target} = "$tgt_dirname/$rdb->{src}";
	}
    }
unknown's avatar
unknown committed
346 347 348 349 350 351 352
    else
    {
      die "Last argument ($tgt_dirname) is not a directory\n"
	if (!(-e $tgt_dirname && -d $tgt_dirname ) );
      foreach my $rdb ( @db_desc ) {
	$rdb->{target} = "$tgt_dirname/$rdb->{src}";
      }
unknown's avatar
unknown committed
353
    }
unknown's avatar
unknown committed
354 355 356
  }
else {
  die "Error: expected \$opt{suffix} to exist" unless ( exists $opt{suffix} );
unknown's avatar
unknown committed
357

unknown's avatar
unknown committed
358 359 360
  foreach my $rdb ( @db_desc ) {
    $rdb->{target} = "$datadir/$rdb->{src}$opt{suffix}";
  }
unknown's avatar
unknown committed
361 362 363 364 365 366 367 368
}

print Dumper( \@db_desc ) if ( $opt{debug} );

# --- bail out if all specified databases are empty ---

die "No tables to hot-copy" unless ( length $hc_locks );

unknown's avatar
unknown committed
369
# --- create target directories if we are using 'cp' ---
unknown's avatar
unknown committed
370 371

my @existing = ();
unknown's avatar
unknown committed
372 373 374 375

if ($opt{method} =~ /^cp\b/)
{
  foreach my $rdb ( @db_desc ) {
unknown's avatar
unknown committed
376
    push @existing, $rdb->{target} if ( -d  $rdb->{target} );
unknown's avatar
unknown committed
377
  }
unknown's avatar
unknown committed
378

unknown's avatar
unknown committed
379 380 381 382 383
  if ( @existing && !$opt{allowold} )
  {
    $dbh->disconnect();
    die "Can't hotcopy to '", join( "','", @existing ), "' because directory\nalready exist and the --allowold option was not given.\n"
  }
unknown's avatar
unknown committed
384
}
unknown's avatar
unknown committed
385 386 387 388

retire_directory( @existing ) if ( @existing );

foreach my $rdb ( @db_desc ) {
unknown's avatar
unknown committed
389 390 391
    foreach my $td ( '', @{$rdb->{raid_dirs}} ) {

	my $tgt_dirpath = "$rdb->{target}/$td";
392 393
	# Remove trailing slashes (needed for Mac OS X)
    	substr($tgt_dirpath, 1) =~ s|/+$||;
unknown's avatar
unknown committed
394 395 396 397 398 399 400 401 402 403 404
	if ( $opt{dryrun} ) {
	    print "mkdir $tgt_dirpath, 0750\n";
	}
	elsif ($opt{method} =~ /^scp\b/) {
	    ## assume it's there?
	    ## ...
	}
	else {
	    mkdir($tgt_dirpath, 0750)
		or die "Can't create '$tgt_dirpath': $!\n";
	}
unknown's avatar
unknown committed
405 406 407 408 409 410 411 412 413 414 415 416 417
    }
}

##############################
# --- PERFORM THE HOT-COPY ---
#
# Note that we try to keep the time between the LOCK and the UNLOCK
# as short as possible, and only start when we know that we should
# be able to complete without error.

# read lock all the tables we'll be copying
# in order to get a consistent snapshot of the database

418 419 420 421 422 423
if ( $opt{checkpoint} || $opt{record_log_pos} ) {
  # convert existing READ lock on checkpoint and/or log_pos table into WRITE lock
  foreach my $table ( grep { defined } ( $opt{checkpoint}, $opt{record_log_pos} ) ) {
    $hc_locks .= ", $table WRITE" 
	unless ( $hc_locks =~ s/$table\s+READ/$table WRITE/ );
  }
unknown's avatar
unknown committed
424 425 426 427 428 429
}

my $hc_started = time;	# count from time lock is granted

if ( $opt{dryrun} ) {
    print "LOCK TABLES $hc_locks\n";
430
    print "FLUSH TABLES /*!32323 $hc_tables */\n";
unknown's avatar
unknown committed
431
    print "FLUSH LOGS\n" if ( $opt{flushlog} );
unknown's avatar
unknown committed
432 433
    print "RESET MASTER\n" if ( $opt{resetmaster} );
    print "RESET SLAVE\n" if ( $opt{resetslave} );
unknown's avatar
unknown committed
434 435 436 437 438 439 440 441 442
}
else {
    my $start = time;
    $dbh->do("LOCK TABLES $hc_locks");
    printf "Locked $num_tables tables in %d seconds.\n", time-$start unless $opt{quiet};
    $hc_started = time;	# count from time lock is granted

    # flush tables to make on-disk copy uptodate
    $start = time;
443 444
    $dbh->do("FLUSH TABLES /*!32323 $hc_tables */");
    printf "Flushed tables ($hc_tables) in %d seconds.\n", time-$start unless $opt{quiet};
unknown's avatar
unknown committed
445
    $dbh->do( "FLUSH LOGS" ) if ( $opt{flushlog} );
unknown's avatar
unknown committed
446 447
    $dbh->do( "RESET MASTER" ) if ( $opt{resetmaster} );
    $dbh->do( "RESET SLAVE" ) if ( $opt{resetslave} );
448 449 450 451 452

    if ( $opt{record_log_pos} ) {
	record_log_pos( $dbh, $opt{record_log_pos} );
	$dbh->do("FLUSH TABLES /*!32323 $hc_tables */");
    }
unknown's avatar
unknown committed
453
}
unknown's avatar
unknown committed
454

unknown's avatar
unknown committed
455 456
my @failed = ();

unknown's avatar
unknown committed
457 458 459 460 461
foreach my $rdb ( @db_desc )
{
  my @files = map { "$datadir/$rdb->{src}/$_" } @{$rdb->{files}};
  next unless @files;
  
unknown's avatar
unknown committed
462
  eval { copy_files($opt{method}, \@files, $rdb->{target}, $rdb->{raid_dirs} ); };
unknown's avatar
unknown committed
463 464 465
  push @failed, "$rdb->{src} -> $rdb->{target} failed: $@"
    if ( $@ );
  
unknown's avatar
unknown committed
466
  @files = @{$rdb->{index}};
unknown's avatar
unknown committed
467 468
  if ($rdb->{index})
  {
unknown's avatar
unknown committed
469 470
    copy_index($opt{method}, \@files,
	       "$datadir/$rdb->{src}", $rdb->{target} );
unknown's avatar
unknown committed
471 472 473 474 475 476 477 478 479 480 481 482 483
  }
  
  if ( $opt{checkpoint} ) {
    my $msg = ( $@ ) ? "Failed: $@" : "Succeeded";
    
    eval {
      $dbh->do( qq{ insert into $opt{checkpoint} (src, dest, msg) 
		      VALUES ( '$rdb->{src}', '$rdb->{target}', '$msg' )
		    } ); 
    };
    
    if ( $@ ) {
      warn "Failed to update checkpoint table: $@\n";
unknown's avatar
unknown committed
484
    }
unknown's avatar
unknown committed
485
  }
unknown's avatar
unknown committed
486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514
}

if ( $opt{dryrun} ) {
    print "UNLOCK TABLES\n";
    if ( @existing && !$opt{keepold} ) {
	my @oldies = map { $_ . '_old' } @existing;
	print "rm -rf @oldies\n" 
    }
    $dbh->disconnect();
    exit(0);
}
else {
    $dbh->do("UNLOCK TABLES");
}

my $hc_dur = time - $hc_started;
printf "Unlocked tables.\n" unless $opt{quiet};

#
# --- HOT-COPY COMPLETE ---
###########################

$dbh->disconnect;

if ( @failed ) {
    # hotcopy failed - cleanup
    # delete any @targets 
    # rename _old copy back to original

515 516 517 518 519 520
    my @targets = ();
    foreach my $rdb ( @db_desc ) {
        push @targets, $rdb->{target} if ( -d  $rdb->{target} );
    }
    print "Deleting @targets \n" if $opt{debug};

unknown's avatar
unknown committed
521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554
    print "Deleting @targets \n" if $opt{debug};
    rmtree([@targets]);
    if (@existing) {
	print "Restoring @existing from back-up\n" if $opt{debug};
        foreach my $dir ( @existing ) {
	    rename("${dir}_old", $dir )
	      or warn "Can't rename ${dir}_old to $dir: $!\n";
	}
    }

    die join( "\n", @failed );
}
else {
    # hotcopy worked
    # delete _old unless $opt{keepold}

    if ( @existing && !$opt{keepold} ) {
	my @oldies = map { $_ . '_old' } @existing;
	print "Deleting previous copy in @oldies\n" if $opt{debug};
	rmtree([@oldies]);
    }

    printf "$0 copied %d tables (%d files) in %d second%s (%d seconds overall).\n",
	    $num_tables, $num_files,
	    $hc_dur, ($hc_dur==1)?"":"s", time - $start_time
	unless $opt{quiet};
}

exit 0;


# ---

sub copy_files {
unknown's avatar
unknown committed
555
    my ($method, $files, $target, $raid_dirs) = @_;
unknown's avatar
unknown committed
556 557
    my @cmd;
    print "Copying ".@$files." files...\n" unless $opt{quiet};
unknown's avatar
unknown committed
558

unknown's avatar
unknown committed
559
    if ($method =~ /^s?cp\b/) { # cp or scp with optional flags
unknown's avatar
unknown committed
560
	my @cp = ($method);
unknown's avatar
unknown committed
561 562
	# add option to preserve mod time etc of copied files
	# not critical, but nice to have
563
	push @cp, "-p" if $^O =~ m/^(solaris|linux|freebsd|darwin)$/;
unknown's avatar
unknown committed
564 565

	# add recursive option for scp
566
	push @cp, "-r" if $^O =~ /m^(solaris|linux|freebsd|darwin)$/ && $method =~ /^scp\b/;
unknown's avatar
unknown committed
567

unknown's avatar
unknown committed
568
	my @non_raid = map { "'$_'" } grep { ! m:/\d{2}/[^/]+$: } @$files;
unknown's avatar
unknown committed
569

unknown's avatar
unknown committed
570
	# add files to copy and the destination directory
571
	safe_system( @cp, @non_raid, "'$target'" );
unknown's avatar
unknown committed
572 573
	
	foreach my $rd ( @$raid_dirs ) {
unknown's avatar
unknown committed
574 575
	    my @raid = map { "'$_'" } grep { m:$rd/: } @$files;
	    safe_system( @cp, @raid, "'$target'/$rd" ) if ( @raid );
unknown's avatar
unknown committed
576
	}
unknown's avatar
unknown committed
577
    }
unknown's avatar
unknown committed
578 579
    else
    {
unknown's avatar
unknown committed
580 581
	die "Can't use unsupported method '$method'\n";
    }
unknown's avatar
unknown committed
582
}
unknown's avatar
unknown committed
583

unknown's avatar
unknown committed
584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636
#
# Copy only the header of the index file
#

sub copy_index
{
  my ($method, $files, $source, $target) = @_;
  my $tmpfile="$opt_tmpdir/mysqlhotcopy$$";
  
  print "Copying indices for ".@$files." files...\n" unless $opt{quiet};  
  foreach my $file (@$files)
  {
    my $from="$source/$file";
    my $to="$target/$file";
    my $buff;
    open(INPUT, "<$from") || die "Can't open file $from: $!\n";
    my $length=read INPUT, $buff, 2048;
    die "Can't read index header from $from\n" if ($length < 1024);
    close INPUT;
    
    if ( $opt{dryrun} )
    {
      print "$opt{method}-header $from $to\n";
    }
    elsif ($opt{method} eq 'cp')
    {
      open(OUTPUT,">$to")   || die "Can\'t create file $to: $!\n";
      if (syswrite(OUTPUT,$buff) != length($buff))
      {
	die "Error when writing data to $to: $!\n";
      }
      close OUTPUT	   || die "Error on close of $to: $!\n";
    }
    elsif ($opt{method} eq 'scp')
    {
      my $tmp=$tmpfile;
      open(OUTPUT,">$tmp") || die "Can\'t create file $tmp: $!\n";
      if (syswrite(OUTPUT,$buff) != length($buff))
      {
	die "Error when writing data to $tmp: $!\n";
      }
      close OUTPUT	     || die "Error on close of $tmp: $!\n";
      safe_system("scp $tmp $to");
    }
    else
    {
      die "Can't use unsupported method '$opt{method}'\n";
    }
  }
  unlink "$tmpfile" if  ($opt{method} eq 'scp');
}


unknown's avatar
unknown committed
637 638
sub safe_system
{
unknown's avatar
unknown committed
639
  my @cmd= @_;
unknown's avatar
unknown committed
640

unknown's avatar
unknown committed
641 642 643 644 645 646 647 648
  if ( $opt{dryrun} )
  {
    print "@cmd\n";
    return;
  }

  ## for some reason system fails but backticks works ok for scp...
  print "Executing '@cmd'\n" if $opt{debug};
unknown's avatar
unknown committed
649
  my $cp_status = system "@cmd > /dev/null";
unknown's avatar
unknown committed
650 651 652 653 654
  if ($cp_status != 0) {
    warn "Burp ('scuse me). Trying backtick execution...\n" if $opt{debug}; #'
    ## try something else
    `@cmd` && die "Error: @cmd failed ($cp_status) while copying files.\n";
  }
unknown's avatar
unknown committed
655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677
}

sub retire_directory {
    my ( @dir ) = @_;

    foreach my $dir ( @dir ) {
	my $tgt_oldpath = $dir . '_old';
	if ( $opt{dryrun} ) {
	    print "rmtree $tgt_oldpath\n" if ( -d $tgt_oldpath );
	    print "rename $dir, $tgt_oldpath\n";
	    next;
	}

	if ( -d $tgt_oldpath ) {
	    print "Deleting previous 'old' hotcopy directory ('$tgt_oldpath')\n" unless $opt{quiet};
	    rmtree([$tgt_oldpath])
	}
	rename($dir, $tgt_oldpath)
	  or die "Can't rename $dir=>$tgt_oldpath: $!\n";
	print "Existing hotcopy directory renamed to '$tgt_oldpath'\n" unless $opt{quiet};
    }
}

678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711
sub record_log_pos {
    my ( $dbh, $table_name ) = @_;

    eval {
	my ($file,$position) = get_row( $dbh, "show master status" );
	die "master status is undefined" if !defined $file || !defined $position;
	
	my ($master_host, undef, undef, undef, $log_file, $log_pos ) 
	    = get_row( $dbh, "show slave status" );
	
	my $hostname = hostname();
	
	$dbh->do( qq{ replace into $table_name 
			  set host=?, log_file=?, log_pos=?, 
                          master_host=?, master_log_file=?, master_log_pos=? }, 
		  undef, 
		  $hostname, $file, $position, 
		  $master_host, $log_file, $log_pos  );
	
    };
    
    if ( $@ ) {
	warn "Failed to store master position: $@\n";
    }
}

sub get_row {
  my ( $dbh, $sql ) = @_;

  my $sth = $dbh->prepare($sql);
  $sth->execute;
  return $sth->fetchrow_array();
}

unknown's avatar
unknown committed
712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740
sub scan_raid_dir {
    my ( $r_db_files, $data_dir, @raid_dir ) = @_;

    local(*RAID_DIR);
    
    foreach my $rd ( @raid_dir ) {

	opendir(RAID_DIR, "$data_dir/$rd" ) 
	    or die "Cannot open dir '$data_dir/$rd': $!";

	while ( defined( my $name = readdir RAID_DIR ) ) {
	    $r_db_files->{"$rd/$name"} = $1 if ( $name =~ /(.+)\.\w+$/ );
	}
	closedir( RAID_DIR );
    }
}

sub get_raid_dirs {
    my ( $r_files ) = @_;

    my %dirs = ();
    foreach my $f ( @$r_files ) {
	if ( $f =~ m:^(\d\d)/: ) {
	    $dirs{$1} = 1;
	}
    }
    return sort keys %dirs;
}

unknown's avatar
unknown committed
741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759
sub get_list_of_tables {
    my ( $db ) = @_;

    # "use database" cannot cope with database names containing spaces
    # so create a new connection 

    my $dbh = DBI->connect("dbi:mysql:${db}${dsn};mysql_read_default_group=mysqlhotcopy",
			    $opt{user}, $opt{password},
    {
	RaiseError => 1,
	PrintError => 0,
	AutoCommit => 1,
    });

    my @dbh_tables = eval { $dbh->tables() };
    $dbh->disconnect();
    return @dbh_tables;
}

unknown's avatar
unknown committed
760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793
__END__

=head1 DESCRIPTION

mysqlhotcopy is designed to make stable copies of live MySQL databases.

Here "live" means that the database server is running and the database
may be in active use. And "stable" means that the copy will not have
any corruptions that could occur if the table files were simply copied
without first being locked and flushed from within the server.

=head1 OPTIONS

=over 4

=item --checkpoint checkpoint-table

As each database is copied, an entry is written to the specified
checkpoint-table.  This has the happy side-effect of updating the
MySQL update-log (if it is switched on) giving a good indication of
where roll-forward should begin for backup+rollforward schemes.

The name of the checkpoint table should be supplied in database.table format.
The checkpoint-table must contain at least the following fields:

=over 4

  time_stamp timestamp not null
  src varchar(32)
  dest varchar(60)
  msg varchar(255)

=back

794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825
=item --record_log_pos log-pos-table

Just before the database files are copied, update the record in the
log-pos-table from the values returned from "show master status" and
"show slave status". The master status values are stored in the
log_file and log_pos columns, and establish the position in the binary
logs that any slaves of this host should adopt if initialised from
this dump.  The slave status values are stored in master_host,
master_log_file, and master_log_pos, and these are useful if the host
performing the dump is a slave and other sibling slaves are to be
initialised from this dump.

The name of the log-pos table should be supplied in database.table format.
A sample log-pos table definition:

=over 4

CREATE TABLE log_pos (
  host            varchar(60) NOT null,
  time_stamp      timestamp(14) NOT NULL,
  log_file        varchar(32) default NULL,
  log_pos         int(11)     default NULL,
  master_host     varchar(60) NULL,
  master_log_file varchar(32) NULL,
  master_log_pos  int NULL,

  PRIMARY KEY  (host) 
);

=back


unknown's avatar
unknown committed
826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854
=item --suffix suffix

Each database is copied back into the originating datadir under
a new name. The new name is the original name with the suffix
appended. 

If only a single db_name is supplied and the --suffix flag is not
supplied, then "--suffix=_copy" is assumed.

=item --allowold

Move any existing version of the destination to a backup directory for
the duration of the copy. If the copy successfully completes, the backup 
directory is deleted - unless the --keepold flag is set.  If the copy fails,
the backup directory is restored.

The backup directory name is the original name with "_old" appended.
Any existing versions of the backup directory are deleted.

=item --keepold

Behaves as for the --allowold, with the additional feature 
of keeping the backup directory after the copy successfully completes.

=item --flushlog

Rotate the log files by executing "FLUSH LOGS" after all tables are
locked, and before they are copied.

unknown's avatar
unknown committed
855 856 857 858 859 860 861 862 863 864 865 866
=item --resetmaster

Reset the bin-log by executing "RESET MASTER" after all tables are
locked, and before they are copied. Usefull if you are recovering a
slave in a replication setup.

=item --resetslave

Reset the master.info by executing "RESET SLAVE" after all tables are
locked, and before they are copied. Usefull if you are recovering a
server in a mutual replication setup.

unknown's avatar
unknown committed
867 868
=item --regexp pattern

unknown's avatar
unknown committed
869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884
Copy all databases with names matching the pattern

=item db_name./pattern/

Copy only tables matching pattern. Shell metacharacters ( (, ), |, !,
etc.) have to be escaped (e.g. \). For example, to select all tables
in database db1 whose names begin with 'foo' or 'bar':

    mysqlhotcopy --indices --method=cp db1./^\(foo\|bar\)/

=item db_name./~pattern/

Copy only tables not matching pattern. For example, to copy tables
that do not begin with foo nor bar:

    mysqlhotcopy --indices --method=cp db1./~^\(foo\|bar\)/
unknown's avatar
unknown committed
885 886 887 888 889 890 891 892 893 894 895 896 897

=item -?, --help

Display helpscreen and exit

=item -u, --user=#         

user for database login if not current user

=item -p, --password=#     

password to use when connecting to server

898 899 900 901 902
=item -h, -h, --host=#

Hostname for local server when connecting over TCP/IP.  By specifying this
different from 'localhost' will trigger mysqlhotcopy to use TCP/IP connection.

unknown's avatar
unknown committed
903 904
=item -P, --port=#         

905 906
port to use when connecting to MySQL server with TCP/IP.  This is only used
when using the --host option.
unknown's avatar
unknown committed
907 908 909 910 911

=item -S, --socket=#         

UNIX domain socket to use when connecting to local server

unknown's avatar
unknown committed
912
=item  --noindices          
unknown's avatar
unknown committed
913

unknown's avatar
unknown committed
914 915 916
Don\'t include index files in copy. Only up to the first 2048 bytes
are copied;  You can restore the indexes with isamchk -r or myisamchk -r
on the backup.
unknown's avatar
unknown committed
917 918 919

=item  --method=#           

unknown's avatar
unknown committed
920 921 922 923 924
method for copy (only "cp" currently supported). Alpha support for
"scp" was added in November 2000. Your experience with the scp method
will vary with your ability to understand how scp works. 'man scp'
and 'man ssh' are your friends.

unknown's avatar
unknown committed
925 926 927 928
The destination directory _must exist_ on the target machine using the
scp method. --keepold and --allowold are meeningless with scp.
Liberal use of the --debug option will help you figure out what\'s
really going on when you do an scp.
unknown's avatar
unknown committed
929 930 931 932 933

Note that using scp will lock your tables for a _long_ time unless
your network connection is _fast_. If this is unacceptable to you,
use the 'cp' method to copy the tables to some temporary area and then
scp or rsync the files at your leisure.
unknown's avatar
unknown committed
934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956

=item -q, --quiet              

be silent except for errors

=item  --debug

Debug messages are displayed 

=item -n, --dryrun

Display commands without actually doing them

=back

=head1 WARRANTY

This software is free and comes without warranty of any kind. You
should never trust backup software without studying the code yourself.
Study the code inside this script and only rely on it if I<you> believe
that it does the right thing for you.

Patches adding bug fixes, documentation and new features are welcome.
957
Please send these to internals@lists.mysql.com.
unknown's avatar
unknown committed
958 959 960

=head1 TO DO

unknown's avatar
unknown committed
961 962
Extend the individual table copy to allow multiple subsets of tables
to be specified on the command line:
unknown's avatar
unknown committed
963 964 965 966 967 968 969

  mysqlhotcopy db newdb  t1 t2 /^foo_/ : t3 /^bar_/ : +

where ":" delimits the subsets, the /^foo_/ indicates all tables
with names begining with "foo_" and the "+" indicates all tables
not copied by the previous subsets.

unknown's avatar
unknown committed
970 971 972
newdb is either another not existing database or a full path to a directory
where we can create a directory 'db'

unknown's avatar
unknown committed
973
Add option to lock each table in turn for people who don\'t need
unknown's avatar
unknown committed
974 975 976 977 978 979 980 981 982 983 984 985 986
cross-table integrity.

Add option to FLUSH STATUS just before UNLOCK TABLES.

Add support for other copy methods (eg tar to single file?).

Add support for forthcoming MySQL ``RAID'' table subdirectory layouts.

=head1 AUTHOR

Tim Bunce

Martin Waite - added checkpoint, flushlog, regexp and dryrun options
987 988
               Fixed cleanup of targets when hotcopy fails. 
	       Added --record_log_pos.
unknown's avatar
unknown committed
989
               RAID tables are now copied (don't know if this works over scp).
unknown's avatar
unknown committed
990

unknown's avatar
unknown committed
991 992 993
Ralph Corderoy - added synonyms for commands

Scott Wiersdorf - added table regex and scp support
unknown's avatar
unknown committed
994 995

Monty - working --noindex (copy only first 2048 bytes of index file)
unknown's avatar
unknown committed
996
        Fixes for --method=scp
unknown's avatar
unknown committed
997 998

Ask Bjoern Hansen - Cleanup code to fix a few bugs and enable -w again.
unknown's avatar
unknown committed
999 1000 1001

Emil S. Hansen - Added resetslave and resetmaster.

unknown's avatar
unknown committed
1002 1003 1004 1005
Jeremy D. Zawodny - Removed depricated DBI calls.  Fixed bug which
resulted in nothing being copied when a regexp was specified but no
database name(s).

unknown's avatar
unknown committed
1006
Martin Waite - Fix to handle database name that contains space.
1007 1008

Paul DuBois - Remove end '/' from directory names