diff --git a/mysql-test/README.stress b/mysql-test/README.stress new file mode 100644 index 0000000000000000000000000000000000000000..001ecceef1b5fec6a2b76411b23c8ee721f860c3 --- /dev/null +++ b/mysql-test/README.stress @@ -0,0 +1,116 @@ + +Overview +-------- + +Stress script is designed to perform testsing of mysql server in +multi-thread environment. + +Stress script allows: + + - to use for stress testing mysqltest binary as test engine + - to use for stress testing both regular test suite and any + additional test suites (e.g. mysql-test-extra-5.0) + - to specify files with lists of tests both for initialization of + stress db and for further testing itself + - to define number of threads that will be concurrently used in testing + - to define limitations for test run. e.g. number of tests or loops + for execution or duration of testing, delay between test executions, etc. + - to get readable log file which can be used for identification of + errors arose during testing + +All functionality regarding stress testing was implemeted in +mysql-stress-test.pl script and there are two ways to run stress test: + + - for most cases it is enough to use options below for starting of + stress test from mysql-test-run wrapper. In this case server will + be run automatically, all preparation steps will be performed + and after that stress test will be started. + + - in advanced case one can run mysql-stress-test.pl script directly. + But it requires to perform some preparation steps and to specify a + bunch of options as well so this way may look a bit complicate. + +Usage +----- + +Below is list of stress test specific options for mysql-test-run: + +--stress + Enable stress mode + +--stress-suite=<suite name> + Test suite name that will be used in stress testing. + We assume that all suites are located in mysql-test/suite directory + There is one special suite name - <main|default> that corresponds + to regular test suite located in mysql-test directory. + +--stress-threads=<number of threads> + Number of threads that will be used in stress testing + +--stress-tests-file=<filename with list of tests> + Filename with list of tests(without .test suffix) that will be used in + stress testing. Default filename is stress_tests.txt and default + location of this file is suite/<suite name>/stress_tests.txt + +--stress-init-file=<filename with list of tests> + Filename with list of tests(without .test suffix) that will be used in + stress testing for initialization of stress db. These tests will be + executed only once before starting of test itself. Default filename + is stress_init.txt and default location of this file is + suite/<suite name>/stress_init.txt + +--stress-mode=<method which will be used for choosing tests from the list> + Possible values are: random(default), seq + + There are two possible modes which affect order of selecting of tests + from the list: + - in random mode tests will be selected in random order + - in seq mode each thread will execute tests in the loop one by one as + they specified in the list file. + +--stress-test-count= <number> + Total number of tests that will be executed concurrently by all threads + +--stress-loop-count= <number> + Total number of loops in seq mode that will be executed concurrently + by all threads + +--stress-test-duration= <number> + Duration of stress testing in seconds + +Examples +-------- + +1. Example of simple command line to start stress test: + + mysql-test-run --stress alias + +Runs stress test with default values for number of threads and number of tests, +with test 'alias' from suite 'main'. + +2. Using in stress testing tests from other suites: + + - mysql-test-run --stress --stress-threads=10 --stress-test-count=1000 \ + --stress-suite=example --stress-tests-file=testslist.txt + + Will run stress test with 10 threads, will execute 1000 tests by all + threads, test will be used from suite 'example', list of test will be + taken from file 'testslist.txt' + + - mysql-test-run --stress --stress-threads=10 --stress-test-count=1000 \ + --stress-suite=example sum_distinct + + Will run stress test with 10 threads, will execute 1000 tests by all + threads, test will be used from suite 'example', list of test contains + only one test 'sum_distinct' + +3. Debugging of issues found with stress test + + Right now stress test is not fully integrated in mysql-test-run + and does not support --gdb option so to debug issue found with stress + test you have to start separately mysql server under debuger and then + run stress test as: + + - mysql-test-run --extern --stress --stress-threads=10 \ + --stress-test-count=1000 --stress-suite=example \ + sum_distinct diff --git a/mysql-test/mysql-stress-test.pl b/mysql-test/mysql-stress-test.pl new file mode 100755 index 0000000000000000000000000000000000000000..899dd06a74611ba325ed0c5bb9317b3f71239ba3 --- /dev/null +++ b/mysql-test/mysql-stress-test.pl @@ -0,0 +1,1148 @@ +#!/usr/bin/perl +# ====================================================================== +# MySQL server stress test system +# ====================================================================== +# +########################################################################## +# +# SCENARIOS AND REQUIREMENTS +# +# The system should perform stress testing of MySQL server with +# following requirements and basic scenarios: +# +# Basic requirements: +# +# Design of stress script should allow one: +# +# - to use for stress testing mysqltest binary as test engine +# - to use for stress testing both regular test suite and any +# additional test suites (e.g. mysql-test-extra-5.0) +# - to specify files with lists of tests both for initialization of +# stress db and for further testing itself +# - to define number of threads that will be concurrently used in testing +# - to define limitations for test run. e.g. number of tests or loops +# for execution or duration of testing, delay between test executions, etc. +# - to get readable log file which can be used for identification of +# errors arose during testing +# +# Basic scenarios: +# +# * It should be possible to run stress script in standalone mode +# which will allow to create various scenarios of stress workloads: +# +# simple ones: +# +# box #1: +# - one instance of script with list of tests #1 +# +# and more advanced ones: +# +# box #1: +# - one instance of script with list of tests #1 +# - another instance of script with list of tests #2 +# box #2: +# - one instance of script with list of tests #3 +# - another instance of script with list of tests #4 +# that will recreate whole database to back it to clean +# state +# +# One kind of such complex scenarios maybe continued testing +# when we want to run stress tests from many boxes with various +# lists of tests that will last very long time. And in such case +# we need some wrapper for MySQL server that will restart it in +# case of crashes. +# +# * It should be possible to run stress script in ad-hoc mode from +# shell or perl versions of mysql-test-run. This allows developers +# to reproduce and debug errors that was found in continued stress +# testing +# +######################################################################## + +use Config; + +if (!defined($Config{useithreads})) +{ + die <<EOF; +It is unable to run threaded version of stress test on this system +due to disabled ithreads. Please check that installed perl binary +was built with support of ithreads. +EOF +} + +use threads; +use threads::shared; + +use IO::Socket; +use Sys::Hostname; +use File::Copy; +use File::Spec; +use File::Find; +use File::Basename; +use File::Path; +use Cwd; + +use Data::Dumper; +use Getopt::Long; + +my $stress_suite_version="1.0"; + +$|=1; + +$opt_server_host=""; +$opt_server_logs_dir=""; +$opt_help=""; +$opt_server_port=""; +$opt_server_socket=""; +$opt_server_user=""; +$opt_server_password=""; +$opt_server_database=""; +$opt_cleanup=""; +$opt_verbose=""; +$opt_log_error_details=""; + + +$opt_suite="main"; +$opt_stress_suite_basedir=""; +$opt_stress_basedir=""; +$opt_stress_datadir=""; +$opt_test_suffix=""; + +$opt_stress_mode="random"; + +$opt_loop_count=0; +$opt_test_count=0; +$opt_test_duration=0; +$opt_abort_on_error=0; +$opt_sleep_time = 0; +$opt_threads=1; +$pid_file="mysql_stress_test.pid"; +$opt_mysqltest= ($^O =~ /mswin32/i) ? "mysqltest.exe" : "mysqltest"; +$opt_check_tests_file=""; +@mysqltest_args=("--silent", "-v", "--skip-safemalloc"); + +# Client ip address +$client_ip=inet_ntoa((gethostbyname(hostname()))[4]); +$client_ip=~ s/\.//g; + +%tests_files=(client => {mtime => 0, data => []}, + initdb => {mtime => 0, data => []}); + +# Error codes and sub-strings with corresponding severity +# +# S1 - Critical errors - cause immediately abort of testing. These errors +# could be caused by server crash or impossibility +# of test execution +# +# S2 - Serious errors - these errors are bugs for sure as it knowns that +# they shouldn't appear during stress testing +# +# S3 - Non-seriuos errros - these errors could be caused by fact that +# we execute simultaneously statements that +# affect tests executed by other threads + +%error_strings = ( 'Failed in mysql_real_connect()' => S1, + 'not found (Errcode: 2)' => S1 ); + +%error_codes = ( 1012 => S2, 1015 => S2, 1021 => S2, + 1027 => S2, 1037 => S2, 1038 => S2, + 1039 => S2, 1040 => S2, 1046 => S2, + 1180 => S2, 1181 => S2, 1203 => S2, + 1205 => S2, 1206 => S2, 1207 => S2, + 1223 => S2, 2013 => S1); + +share(%test_counters); +%test_counters=( loop_count => 0, test_count=>0); + +share($exiting); +$exiting=0; + +share($test_counters_lock); +$test_counters_lock=0; +share($log_file_lock); +$log_file_lock=0; + +$SIG{INT}= \&sig_INT_handler; +$SIG{TERM}= \&sig_TERM_handler; + + +GetOptions("server-host=s", "server-logs-dir=s", "server-port=s", + "server-socket=s", "server-user=s", "server-password=s", + "server-database=s", + "stress-suite-basedir=s", "suite=s", "stress-init-file:s", + "stress-tests-file:s", "stress-basedir=s", "stress-mode=s", + "stress-datadir=s", + "threads=s", "sleep-time=s", "loop-count=i", "test-count=i", + "test-duration=i", "test-suffix=s", "check-tests-file", + "verbose", "log-error-details", "cleanup", "mysqltest=s", + "abort-on-error", "help") || usage(); + +usage() if ($opt_help); + +#$opt_abort_on_error=1; + +$test_dirname=get_timestamp(); +$test_dirname.="-$opt_test_suffix" if ($opt_test_suffix ne ''); + +print <<EOF; +############################################################# + CONFIGURATION STAGE +############################################################# +EOF + +if ($opt_stress_basedir eq '' || $opt_stress_suite_basedir eq '' || + $opt_server_logs_dir eq '') +{ + die <<EOF; + +Options --stress-basedir, --stress-suite-basedir and --server-logs-dir are +required. Please use these options to specify proper basedir for +client, test suite and location of server logs. + +stress-basedir: '$opt_stress_basedir' +stress-suite-basedir: '$opt_stress_suite_basedir' +server-logs-dir: '$opt_server_logs_dir' + +EOF +} + +#Workaround for case when we got relative but not absolute path +$opt_stress_basedir=File::Spec->rel2abs($opt_stress_basedir); +$opt_stress_suite_basedir=File::Spec->rel2abs($opt_stress_suite_basedir); +$opt_server_logs_dir=File::Spec->rel2abs($opt_server_logs_dir); + +if ($opt_stress_datadir ne '') +{ + $opt_stress_datadir=File::Spec->rel2abs($opt_stress_datadir); +} + +if (! -d "$opt_stress_basedir") +{ + die <<EOF; + +Directory '$opt_stress_basedir' does not exist. +Use --stress-basedir option to specify proper basedir for client + +EOF +} + +if (!-d $opt_stress_suite_basedir) +{ + die <<EOF; + +Directory '$opt_stress_suite_basedir' does not exist. +Use --stress-suite-basedir option to specify proper basedir for test suite + +EOF +} + +$test_dataset_dir=$opt_stress_suite_basedir; +if ($opt_stress_datadir ne '') +{ + if (-d $opt_stress_datadir) + { + $test_dataset_dir=$opt_stress_datadir; + + } + else + { + die <<EOF; +Directory '$opt_stress_datadir' not exists. Please specify proper one +with --stress-datadir option. +EOF + } +} + +if ($^O =~ /mswin32/i) +{ + $test_dataset_dir=~ s/\\/\\\\/g; +} +else +{ + $test_dataset_dir.="/"; +} + + + +if (!-d $opt_server_logs_dir) +{ + die <<EOF; + +Directory server-logs-dir '$opt_server_logs_dir' does not exist. +Use --server-logs-dir option to specify proper directory for storing +logs + +EOF +} +else +{ + #Create sub-directory for test session logs + mkpath(File::Spec->catdir($opt_server_logs_dir, $test_dirname), 0, 0755); + #Define filename of global session log file + $stress_log_file=File::Spec->catfile($opt_server_logs_dir, $test_dirname, + "mysql-stress-test.log"); +} + +if ($opt_suite ne '' && $opt_suite ne 'main' && $opt_suite ne 'default') +{ + $test_suite_dir=File::Spec->catdir($opt_stress_suite_basedir, "suite", $opt_suite); +} +else +{ + $test_suite_dir= $opt_stress_suite_basedir; +} + +if (!-d $test_suite_dir) +{ + die <<EOF + +Directory '$test_suite_dir' does not exist. +Use --suite options to specify proper dir for test suite + +EOF +} + +$test_suite_t_path=File::Spec->catdir($test_suite_dir,'t'); +$test_suite_r_path=File::Spec->catdir($test_suite_dir,'r'); + +foreach my $suite_dir ($test_suite_t_path, $test_suite_r_path) +{ + if (!-d $suite_dir) + { + die <<EOF; + +Directory '$suite_dir' does not exist. +Please ensure that you specified proper source location for +test/result files with --stress-suite-basedir option and name +of test suite with --suite option + +EOF + } +} + +$test_t_path=File::Spec->catdir($opt_stress_basedir,'t'); +$test_r_path=File::Spec->catdir($opt_stress_basedir,'r'); + +foreach $test_dir ($test_t_path, $test_r_path) +{ + if (-d $test_dir) + { + if ($opt_cleanup) + { + #Delete existing 't', 'r', 'r/*' subfolders in $stress_basedir + rmtree("$test_dir", 0, 0); + print "Cleanup $test_dir\n"; + } + else + { + die <<EOF; +Directory '$test_dir' already exist. +Please ensure that you specified proper location of working dir +for current test run with --stress-basedir option or in case of staled +directories use --cleanup option to remove ones +EOF + } + } + #Create empty 't', 'r' subfolders that will be filled later + mkpath("$test_dir", 0, 0777); +} + +if (!defined($opt_stress_tests_file) && !defined($opt_stress_init_file)) +{ + die <<EOF; +You should run stress script either with --stress-tests-file or with +--stress-init-file otions. See help for details. +EOF +} + +if (defined($opt_stress_tests_file)) +{ + if ($opt_stress_tests_file eq '') + { + #Default location of file with set of tests for current test run + $tests_files{client}->{filename}= File::Spec->catfile($opt_stress_suite_basedir, + "testslist_client.txt"); + } + else + { + $tests_files{client}->{filename}= $opt_stress_tests_file; + } + + if (!-f $tests_files{client}->{filename}) + { + die <<EOF; + +File '$tests_files{client}->{filename}' with list of tests not exists. +Please ensure that this file exists, readable or specify another one with +--stress-tests-file option. + +EOF + } +} + +if (defined($opt_stress_init_file)) +{ + if ($opt_stress_init_file eq '') + { + #Default location of file with set of tests for current test run + $tests_files{initdb}->{filename}= File::Spec->catfile($opt_stress_suite_basedir, + "testslist_initdb.txt"); + } + else + { + $tests_files{initdb}->{filename}= $opt_stress_init_file; + } + + if (!-f $tests_files{initdb}->{filename}) + { + die <<EOF; + +File '$tests_files{initdb}->{filename}' with list of tests for initialization of database +for stress test not exists. +Please ensure that this file exists, readable or specify another one with +--stress-init-file option. + +EOF + } +} + +if ($opt_stress_mode !~ /^(random|seq)$/) +{ + die <<EOF +Was specified wrong --stress-mode. Correct values 'random' and 'seq'. +EOF +} + +if (open(TEST, "$opt_mysqltest -V |")) +{ + $mysqltest_version=join("",<TEST>); + close(TEST); + print "FOUND MYSQLTEST BINARY: ", $mysqltest_version,"\n"; +} +else +{ + die <<EOF; +ERROR: mysqltest binary $opt_mysqltest not found $!. +You must either specify file location explicitly using --mysqltest +option, or make sure path to mysqltest binary is listed +in your PATH environment variable. +EOF +} + +# +#Adding mysql server specific command line options for mysqltest binary +# +$opt_server_host= $opt_server_host ? $opt_server_host : "localhost"; +$opt_server_port= $opt_server_port ? $opt_server_port : "3306"; +$opt_server_user= $opt_server_user ? $opt_server_user : "root"; +$opt_server_socket= $opt_server_socket ? $opt_server_socket : "/tmp/mysql.sock"; +$opt_server_database= $opt_server_database ? $opt_server_database : "test"; + +unshift @mysqltest_args, "--host=$opt_server_host"; +unshift @mysqltest_args, "--port=$opt_server_port"; +unshift @mysqltest_args, "--user=$opt_server_user"; +unshift @mysqltest_args, "--password=$opt_server_password"; +unshift @mysqltest_args, "--socket=$opt_server_socket"; +unshift @mysqltest_args, "--database=$opt_server_database"; + +#Export variables that could be used in tests +$ENV{MYSQL_TEST_DIR}=$test_dataset_dir; +$ENV{MASTER_MYPORT}=$opt_server_port; +$ENV{MASTER_MYSOCK}=$opt_server_socket; + +print <<EOF; +TEST-SUITE-BASEDIR: $opt_stress_suite_basedir +SUITE: $opt_suite +TEST-BASE-DIR: $opt_stress_basedir +TEST-DATADIR: $test_dataset_dir +SERVER-LOGS-DIR: $opt_server_logs_dir + +THREADS: $opt_threads +TEST-MODE: $opt_stress_mode + +EOF + +#------------------------------------------------------------------------------- +#At this stage we've already checked all needed pathes/files +#and ready to start the test +#------------------------------------------------------------------------------- + +if (defined($opt_stress_tests_file) || defined($opt_stress_init_file)) +{ + print <<EOF; +############################################################# + PREPARATION STAGE +############################################################# +EOF + + #Copy Test files from network share to 't' folder + print "\nCopying Test files from $test_suite_t_path to $test_t_path folder..."; + find({wanted=>\©_test_files, bydepth=>1}, "$test_suite_t_path"); + print "Done\n"; + + #$test_r_path/r0 dir reserved for initdb + $count_start= defined($opt_stress_init_file) ? 0 : 1; + + our $r_folder=''; + print "\nCreating 'r' folder and copying Protocol files to each 'r#' sub-folder..."; + for($count=$count_start; $count <= $opt_threads; $count++) + { + $r_folder = File::Spec->catdir($test_r_path, "r".$count); + mkpath("$r_folder", 0, 0777); + + find(\©_result_files,"$test_suite_r_path"); + } + print "Done\n\n"; +} + +if (defined($opt_stress_init_file)) +{ + print <<EOF; +############################################################# + INITIALIZATION STAGE +############################################################# +EOF + + #Set limits for stress db initialization + %limits=(loop_count => 1, test_count => undef); + + #Read list of tests from $opt_stress_init_file + read_tests_names($tests_files{initdb}); + test_loop($client_ip, 0, 'seq', $tests_files{initdb}); + #print Dumper($tests_files{initdb}),"\n"; + print <<EOF; + +Done initialization of stress database by tests from +$tests_files{initdb}->{filename} file. + +EOF +} + +if (defined($opt_stress_tests_file)) +{ + print <<EOF; +############################################################# + STRESS TEST RUNNING STAGE +############################################################# +EOF + + $exiting=0; + #Read list of tests from $opt_stress_tests_file + read_tests_names($tests_files{client}); + + #Reset current counter and set limits + %test_counters=( loop_count => 0, test_count=>0); + %limits=(loop_count => $opt_loop_count, test_count => $opt_test_count); + + if (($opt_loop_count && $opt_threads > $opt_loop_count) || + ($opt_test_count && $opt_threads > $opt_test_count)) + { + warn <<EOF; + +WARNING: Possible inaccuracies in number of executed loops or + tests because number of threads bigger than number of + loops or tests: + + Threads will be started: $opt_threads + Loops will be executed: $opt_loop_count + Tests will be executed: $opt_test_count + +EOF + } + + #Create threads (number depending on the variable ) + for ($id=1; $id<=$opt_threads && !$exiting; $id++) + { + $thrd[$id] = threads->create("test_loop", $client_ip, $id, + $opt_stress_mode, $tests_files{client}); + + print "main: Thread ID $id TID ",$thrd[$id]->tid," started\n"; + select(undef, undef, undef, 0.5); + } + + if ($opt_test_duration) + { + sleep($opt_test_duration); + kill INT, $$; #Interrupt child threads + } + + #Let other threads to process INT signal + sleep(1); + + for ($id=1; $id<=$opt_threads;$id++) + { + if (defined($thrd[$id])) + { + $thrd[$id]->join(); + } + } + print "EXIT\n"; +} + +sub test_init +{ + my ($env)=@_; + + $env->{session_id}=$env->{ip}."_".$env->{thread_id}; + $env->{r_folder}='r'.$env->{thread_id}; + $env->{screen_logs}=File::Spec->catdir($opt_server_logs_dir, $test_dirname, + "screen_logs", $env->{session_id}); + $env->{reject_logs}=File::Spec->catdir($opt_server_logs_dir, $test_dirname, + "reject_logs", $env->{session_id}); + + mkpath($env->{screen_logs}, 0, 0755) unless (-d $env->{screen_logs}); + mkpath($env->{reject_logs}, 0, 0755) unless (-d $env->{reject_logs}); + + $env->{session_log}= File::Spec->catfile($env->{screen_logs}, $env->{session_id}.".log"); +} + +sub test_execute +{ + my $env= shift; + my $test_name= shift; + + my $g_start= ""; + my $g_end= ""; + my $mysqltest_cmd= ""; + my @mysqltest_test_args=(); + my @stderr=(); + + #Get time stamp + $g_start = get_timestamp(); + $env->{errors}={}; + @{$env->{test_status}}=(); + + my $test_file= $test_name.".test"; + my $result_file= $test_name.".result"; + my $reject_file = $test_name.'.reject'; + my $output_file = $env->{session_id}.'_'.$test_name.'_'.$g_start."_".$env->{test_count}.'.txt'; + + my $test_filename = File::Spec->catfile($test_t_path, $test_file); + my $result_filename = File::Spec->catdir($test_r_path, $env->{r_folder}, $result_file); + my $reject_filename = File::Spec->catdir($test_r_path, $env->{r_folder}, $reject_file); + my $output_filename = File::Spec->catfile($env->{screen_logs}, $output_file); + + + push @mysqltest_test_args, "--basedir=$opt_stress_suite_basedir/", + "--tmpdir=$opt_stress_basedir", + "-x $test_filename", + "-R $result_filename", + "2>$output_filename"; + + $cmd= "$opt_mysqltest --no-defaults ".join(" ", @mysqltest_args)." ". + join(" ", @mysqltest_test_args); + + system($cmd); + + $exit_value = $? >> 8; + $signal_num = $? & 127; + $dumped_core = $? & 128; + + my $tid= threads->self->tid; + + if (-s $output_filename > 0) + { + #Read stderr for further analysis + open (STDERR_LOG, $output_filename) or + warn "Can't open file $output_filename"; + @stderr=<STDERR_LOG>; + close(STDERR_LOG); + + if ($opt_verbose) + { + $session_debug_file="$opt_stress_basedir/error$tid.txt"; + + stress_log($session_debug_file, + "Something wrong happened during execution of this command line:"); + stress_log($session_debug_file, "MYSQLTEST CMD - $cmd"); + stress_log($session_debug_file, "STDERR:".join("",@stderr)); + + stress_log($session_debug_file, "EXIT STATUS:\n1. EXIT: $exit_value \n". + "2. SIGNAL: $signal_num\n". + "3. CORE: $dumped_core\n"); + } + } + + #If something wrong trying to analyse stderr + if ($exit_value || $signal_num) + { + if (@stderr) + { + foreach my $line (@stderr) + { + #FIXME: we should handle case when for one sub-string/code + # we have several different error messages + # Now for both codes/substrings we assume that + # first found message will represent error + + #Check line for error codes + if (($err_msg, $err_code)= $line=~/failed: ((\d+):.+?$)/) + { + if (!exists($error_codes{$err_code})) + { + $severity="S3"; + $err_code=0; + } + else + { + $severity=$error_codes{$err_code}; + } + + if (!exists($env->{errors}->{$severity}->{$err_code})) + { + $env->{errors}->{$severity}->{$err_code}=[0, $err_msg]; + } + $env->{errors}->{$severity}->{$err_code}->[0]++; + $env->{errors}->{$severity}->{total}++; + } + + #Check line for error patterns + foreach $err_string (keys %error_strings) + { + $pattern= quotemeta $err_string; + if ($line =~ /$pattern/i) + { + my $severity= $error_strings{$err_string}; + if (!exists($env->{errors}->{$severity}->{$err_string})) + { + $env->{errors}->{$severity}->{$err_string}=[0, $line]; + } + $env->{errors}->{$severity}->{$err_string}->[0]++; + $env->{errors}->{$severity}->{total}++; + } + } + } + } + else + { + $env->{errors}->{S3}->{'Unknown error'}= + [1,"Unknown error. Nothing was output to STDERR"]; + $env->{errors}->{S3}->{total}=1; + } + } + + # + #FIXME: Here we can perform further analysis of recognized + # error codes + # + + foreach my $severity (sort {$a cmp $b} keys %{$env->{errors}}) + { + my $total=$env->{errors}->{$severity}->{total}; + if ($total) + { + push @{$env->{test_status}}, "Severity $severity: $total"; + $env->{errors}->{total}=+$total; + } + } + + #FIXME: Should we take into account $exit_value here? + # Now we assume that all stringified errors(i.e. errors without + # error codes) which are not exist in %error_string structure + # are OK + if (!$env->{errors}->{total}) + { + push @{$env->{test_status}},"No Errors. Test Passed OK"; + } + + log_session_errors($env, $test_file); + + if (!$exiting && ($signal_num == 2 || $signal_num == 15 || + ($opt_abort_on_error && $env->{errors}->{S1} > 0))) + { + #mysqltest was interrupted with INT or TERM signals or test was + #ran with --abort-on-error option and we got errors with severity S1 + #so we assume that we should cancel testing and exit + $exiting=1; + print STDERR<<EOF; +WARNING: + mysqltest was interrupted with INT or TERM signals or test was + ran with --abort-on-error option and we got errors with severity S1 + (test cann't connect to the server or server crashed) so we assume that + we should cancel testing and exit. Please check log file for this thread + in $stress_log_file or + inspect below output of the last test case executed with mysqltest to + find out cause of error. + + Output of mysqltest: + @stderr + +EOF + } + + if (-e $reject_filename) + { + move_to_logs($env->{reject_logs}, $reject_filename, $reject_file); + } + + if (-e $output_filename) + { + move_to_logs($env->{screen_logs}, $output_filename, $output_file); + } + +} + +sub test_loop +{ + my %client_env=(); + my $test_name=""; + + # KEY for session identification: IP-THREAD_ID + $client_env{ip} = shift; + $client_env{thread_id} = shift; + + $client_env{mode} = shift; + $client_env{tests_file}=shift; + + $client_env{test_seq_idx}=0; + + #Initialize session variables + test_init(\%client_env); + +LOOP: + + while(!$exiting) + { + if ($opt_check_tests_file) + { + #Check if tests_file was modified and reread it in this case + read_tests_names($client_env{tests_file}, 0); + } + + { + lock($test_counters_lock); + + if (($limits{loop_count} && $limits{loop_count} <= $test_counters{loop_count}*1) || + ($limits{test_count} && $limits{test_count} <= $test_counters{test_count}*1) ) + { + $exiting=1; + next LOOP; + } + } + + #Get random file name + if (($test_name = get_test(\%client_env)) ne '') + { + { + lock($test_counters_lock); + + #Save current counters values + $client_env{loop_count}=$test_counters{loop_count}; + $client_env{test_count}=$test_counters{test_count}; + } + #Run test and analyze results + test_execute(\%client_env, $test_name); + + print "test_loop[".$limits{loop_count}.":". + $limits{test_count}." ". + $client_env{loop_count}.":". + $client_env{test_count}."]:". + " TID ".$client_env{thread_id}. + " test: '$test_name' ". + " Errors: ".join(" ",@{$client_env{test_status}}),"\n"; + print "\n"; + } + + sleep($opt_sleep_time) if($opt_sleep_time); + + } +} + +sub move_to_logs ($$$) +{ + my $path_to_logs = shift; + my $src_file = shift; + my $random_filename = shift; + + my $dst_file = File::Spec->catfile($path_to_logs, $random_filename); + + move ($src_file, $dst_file) or warn<<EOF; +ERROR: move_to_logs: File $src_file cannot be moved to $dst_file: $! +EOF +} + +sub copy_test_files () +{ + if (/\.test$/) + { + $src_file = $File::Find::name; + #print "## $File::Find::topdir - $File::Find::dir - $src_file\n"; + + if ($File::Find::topdir eq $File::Find::dir && $src_file !~ /SCCS/) + { + $test_filename = basename($src_file); + $dst_file = File::Spec->catfile($test_t_path, $test_filename); + + copy($src_file, $dst_file) or die "ERROR: copy_test_files: File cannot be copied. $!"; + } + } +} + +sub copy_result_files () +{ + if (/\.result$/) + { + $src_file = $File::Find::name; + + if ($File::Find::topdir eq $File::Find::dir && $src_file !~ /SCCS/) + { + $result_filename = basename($src_file) ; + $dst_file = File::Spec->catfile($r_folder, $result_filename); + + copy($src_file, $dst_file) or die "ERROR: copy_result_files: File cannot be copied. $!"; + } + } +} + +sub get_timestamp +{ + my ($sec,$min,$hour,$mday,$mon,$year,$wday,$ydat,$isdst) = localtime(); + + return sprintf("%04d%02d%02d%02d%02d%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec); +} + +sub read_tests_names +{ + my $tests_file = shift; + my $force_load = shift; + + if ($force_load || ( (stat($tests_file->{filename}))[9] != $tests_file->{mtime}) ) + { + open (TEST, $tests_file->{filename}) || die ("Could not open file <". + $tests_file->{filename}."> $!"); + @{$tests_file->{data}}= grep {!/^[#\r\n]|^$/} map { s/[\r\n]//g; $_ } <TEST>; + + close (TEST); + $tests_file->{mtime}=(stat(_))[9]; + } +} + +sub get_random_test +{ + my $envt=shift; + my $tests= $envt->{tests_file}->{data}; + + my $random = int(rand(@{$tests})); + my $test = $tests->[$random]; + + return $test; +} + +sub get_next_test +{ + my $envt=shift; + my $test; + + if (@{$envt->{tests_file}->{data}}) + { + $test=${$envt->{tests_file}->{data}}[$envt->{test_seq_idx}]; + $envt->{test_seq_idx}++; + } + + #If we reach bound of array, reset seq index and increment loop counter + if ($envt->{test_seq_idx} == scalar(@{$envt->{tests_file}->{data}})) + { + $envt->{test_seq_idx}=0; + { + lock($test_counters_lock); + $test_counters{loop_count}++; + } + } + + return $test; +} + +sub get_test +{ + my $envt=shift; + + { + lock($test_counters_lock); + $test_counters{test_count}++; + } + + if ($envt->{mode} eq 'seq') + { + return get_next_test($envt); + } + elsif ($envt->{mode} eq 'random') + { + return get_random_test($envt); + } +} + +sub stress_log +{ + my ($log_file, $line)=@_; + + { + open(SLOG,">>$log_file") or warn "Error during opening log file $log_file"; + print SLOG $line,"\n"; + close(SLOG); + } +} + +sub log_session_errors +{ + my ($env, $test_name) = @_; + my $line=''; + + { + lock ($log_file_lock); + + #header in the begining of log file + if (!-e $stress_log_file) + { + stress_log($stress_log_file, + "TestID TID Suite TestFileName Found Errors"); + stress_log($stress_log_file, + "======================================================="); + } + + $line=sprintf('%6d %3d %10s %20s %s', $env->{test_count}, threads->self->tid, + $opt_suite, $test_name, + join(",", @{$env->{test_status}})); + + stress_log($stress_log_file, $line); + #stress_log_with_lock($stress_log_file, "\n"); + + if ($opt_log_error_details) + { + foreach $severity (sort {$a cmp $b} keys %{$env->{errors}}) + { + stress_log($stress_log_file, ""); + foreach $error (keys %{$env->{errors}->{$severity}}) + { + if ($error ne 'total') + { + stress_log($stress_log_file, "$severity: Count:". + $env->{errors}->{$severity}->{$error}->[0]. + " Error:". $env->{errors}->{$severity}->{$error}->[1]); + } + } + } + } + } +} + +sub sig_INT_handler +{ + $SIG{INT}= \&sig_INT_handler; + $exiting=1; + print STDERR "$$: Got INT signal-------------------------------------------\n"; + +} + +sub sig_TERM_handler +{ + $SIG{TERM}= \&sig_TERM_handler; + $exiting=1; + print STDERR "$$: Got TERM signal\n"; +} + +sub usage +{ + print <<EOF; + +The MySQL Stress suite Ver $stress_suite_version + +mysql-stress-test.pl --stress-basedir=<dir> --stress-suite-basedir=<dir> --server-logs-dir=<dir> + +--server-host +--server-port +--server-socket +--server-user +--server-password +--server-logs-dir + Directory where all clients session logs will be stored. Usually + this is shared directory associated with server that used + in testing + + Required option. + +--stress-suite-basedir=<dir> + Directory that has r/ t/ subfolders with test/result files + which will be used for testing. Also by default we are looking + in this directory for 'stress-tests.txt' file which contains + list of tests. It is possible to specify other location of this + file with --stress-tests-file option. + + Required option. + +--stress-basedir=<dir> + Working directory for this test run. This directory will be used + as temporary location for results tracking during testing + + Required option. + +--stress-datadir=<dir> + Location of data files used which will be used in testing. + By default we search for these files in <dir>/data where dir + is value of --stress-suite-basedir option. + +--stress-init-file[=/path/to/file with tests for initialization of stress db] + Using of this option allows to perform initialization of database + by execution of test files. List of tests will be taken either from + specified file or if it omited from default file 'stress-init.txt' + located in <--stress-suite-basedir/--suite> dir + +--stress-tests-file[=/path/to/file with tests] + Using of this option allows to run stress test itself. Tests for testing + will be taken either from specified file or if it omited from default + file 'stress-tests.txt' located in <--stress-suite-basedir/--suite> dir + +--stress-mode= [random|seq] + There are two possible modes which affect order of selecting tests + from the list: + - in random mode tests will be selected in random order + - in seq mode each thread will execute tests in the loop one by one as + they specified in the list file. + +--sleep-time=<time in seconds> + Delay between test execution. Could be usefull in continued testsing + when one of instance of stress script perform periodical cleanup or + recreating of some database objects + +--threads=#number of threads + Define number of threads + +--check-tests-file + Check file with list of tests. If file was modified it will force to + reread list of tests. Could be usefull in continued testing for + adding/removing tests without script interruption + +--mysqltest=/path/to/mysqltest binary + +--verbose + +--cleanup + Force to clean up working directory (specified with --stress-basedir) + +--log-error-details + Enable errors details in the global error log file. (Default: off) + +--test-count=<number of executed tests before we have to exit> +--loop-count=<number of executed loops in sequential mode before we have to exit> +--test-duration=<number of seconds that stress test should run> + +Example of tool usage: + +perl mysql-stress-test.pl \ +--stress-suite-basedir=/opt/qa/mysql-test-extra-5.0/mysql-test \ +--stress-basedir=/opt/qa/test \ +--server-logs-dir=/opt/qa/logs \ +--test-count=20 \ +--stress-tests-file=innodb-tests.txt \ +--stress-init-file=innodb-init.txt \ +--threads=5 \ +--suite=funcs_1 \ +--mysqltest=/opt/mysql/mysql-5.0/client/mysqltest \ +--server-user=root \ +--server-database=test \ +--cleanup \ + +EOF +exit(0); +} + + diff --git a/mysql-test/mysql-test-run.sh b/mysql-test/mysql-test-run.sh index bd0851fa8b98e514c6b13ad4ebe42fed17b7b8bd..a4f8f2ac1f99e4163a8c86a309b89c26d41a8995 100644 --- a/mysql-test/mysql-test-run.sh +++ b/mysql-test/mysql-test-run.sh @@ -256,6 +256,17 @@ NDB_MGM_EXTRA_OPTS= NDB_MGMD_EXTRA_OPTS= NDBD_EXTRA_OPTS= +DO_STRESS="" +STRESS_SUITE="main" +STRESS_MODE="random" +STRESS_THREADS=5 +STRESS_TEST_COUNT=20 +STRESS_LOOP_COUNT="" +STRESS_TEST_DURATION="" +STRESS_INIT_FILE="" +STRESS_TEST_FILE="" +STRESS_TEST="" + while test $# -gt 0; do case "$1" in --embedded-server) @@ -344,6 +355,35 @@ while test $# -gt 0; do DO_BENCH=1 NO_SLAVE=1 ;; + --stress) + DO_STRESS=1 + NO_SLAVE=1 + SKIP_SLAVE=1 + ;; + --stress-suite=*) + STRESS_SUITE=`$ECHO "$1" | $SED -e "s;--stress-suite=;;"` + ;; + --stress-threads=*) + STRESS_THREADS=`$ECHO "$1" | $SED -e "s;--stress-threads=;;"` + ;; + --stress-test-file=*) + STRESS_TEST_FILE=`$ECHO "$1" | $SED -e "s;--stress-test-file=;;"` + ;; + --stress-init-file=*) + STRESS_INIT_FILE=`$ECHO "$1" | $SED -e "s;--stress-init-file=;;"` + ;; + --stress-mode=*) + STRESS_MODE=`$ECHO "$1" | $SED -e "s;--stress-mode=;;"` + ;; + --stress-loop-count=*) + STRESS_LOOP_COUNT=`$ECHO "$1" | $SED -e "s;--stress-loop-count=;;"` + ;; + --stress-test-count=*) + STRESS_TEST_COUNT=`$ECHO "$1" | $SED -e "s;--stress-test-count=;;"` + ;; + --stress-test-duration=*) + STRESS_TEST_DURATION=`$ECHO "$1" | $SED -e "s;--stress-test-duration=;;"` + ;; --big*) # Actually --big-test EXTRA_MYSQL_TEST_OPT="$EXTRA_MYSQL_TEST_OPT $1" ;; --compress) @@ -700,7 +740,7 @@ fi # If we should run all tests cases, we will use a local server for that -if [ -z "$1" ] +if [ -z "$1" -a -z "$DO_STRESS" ] then USE_RUNNING_SERVER=0 fi @@ -1189,7 +1229,7 @@ start_master() then CURR_MASTER_MYSQLD_TRACE="$EXTRA_MASTER_MYSQLD_TRACE$1" fi - if [ -z "$DO_BENCH" ] + if [ -z "$DO_BENCH" -a -z "$DO_STRESS" ] then master_args="--no-defaults --log-bin=$MYSQL_TEST_DIR/var/log/master-bin$1 \ --server-id=$id \ @@ -1626,7 +1666,7 @@ run_testcase () stop_master 1 report_current_test $tname start_master - if [ -n "$USE_NDBCLUSTER" -a -z "$DO_BENCH" ] ; then + if [ -n "$USE_NDBCLUSTER" -a -z "$DO_BENCH" -a -z "$DO_STRESS" ] ; then start_master 1 fi TZ=$MY_TZ; export TZ @@ -1642,7 +1682,7 @@ run_testcase () stop_master 1 report_current_test $tname start_master - if [ -n "$USE_NDBCLUSTER" -a -z "$DO_BENCH" ] ; then + if [ -n "$USE_NDBCLUSTER" -a -z "$DO_BENCH" -a -z "$DO_STRESS" ] ; then start_master 1 fi else @@ -1763,6 +1803,125 @@ run_testcase () fi } +run_stress_test() +{ + + STRESS_BASEDIR="$MYSQL_TEST_DIR/var/stress" + + #Clean-up old stress test basedir + if [ -d $STRESS_BASEDIR ] ; then + $RM -rf $STRESS_BASEDIR + fi + #Create stress test basedir + mkdir $STRESS_BASEDIR + + if [ "$STRESS_SUITE" != "main" -a "$STRESS_SUITE" != "default" ] ; then + STRESS_SUITE_DIR="$MYSQL_TEST_DIR/suite/$STRESS_SUITE" + else + STRESS_SUITE_DIR="$MYSQL_TEST_DIR" + fi + + if [ -d "$STRESS_SUITE_DIR" ] ; then + STRESS_SUITE_T_DIR="$STRESS_SUITE_DIR/t" + STRESS_SUITE_R_DIR="$STRESS_SUITE_DIR/r" + #FIXME: check that dirs above are exist + else + echo "Directory $STRESS_SUITE_DIR with test suite doesn't exists. Abort stress testing" + exit 1 + fi + + if [ -n "$STRESS_TEST" ] ; then + STRESS_TEST_FILE="$STRESS_BASEDIR/stress_tests.txt" + echo $STRESS_TEST > $STRESS_TEST_FILE + elif [ -n "$STRESS_TEST_FILE" ] ; then + STRESS_TEST_FILE="$STRESS_SUITE_DIR/$STRESS_TEST_FILE" + if [ ! -f "$STRESS_TEST_FILE" ] ; then + echo "Specified file $STRESS_TEST_FILE with list of tests does not exist" + echo "Please ensure that file exists and has proper permissions" + exit 1 + fi + else + STRESS_TEST_FILE="$STRESS_SUITE_DIR/stress_tests.txt" + if [ ! -f "$STRESS_TEST_FILE" ] ; then + echo "Default file $STRESS_TEST_FILE with list of tests does not exist." + echo "Please use --stress-test-file option to specify custom one or you can" + echo "just specify name of test for testing as last argument in command line" + exit 1 + fi + fi + + if [ -n "$STRESS_INIT_FILE" ] ; then + STRESS_INIT_FILE="$STRESS_SUITE_DIR/$STRESS_INIT_FILE" + if [ ! -f "$STRESS_INIT_FILE" ] ; then + echo "Specified file $STRESS_INIT_FILE with list of tests doesn't exist." + echo "Please ensure that file exists and has proper permissions" + exit 1 + fi + else + STRESS_INIT_FILE="$STRESS_SUITE_DIR/stress_init.txt" + #Check for default init file + if [ ! -f "$STRESS_INIT_FILE" ] ; then + STRESS_INIT_FILE="" + fi + fi + + if [ "$STRESS_MODE" != "random" -a "$STRESS_MODE" != "seq" ] ; then + echo "You specified wrong mode '$STRESS_MODE' for stress test." + echo "Correct values are 'random' or 'seq'" + exit 1 + fi + + STRESS_TEST_ARGS="--server-socket=$MASTER_MYSOCK \ + --server-user=$DBUSER \ + --server-database=$DB \ + --stress-suite-basedir=$MYSQL_TEST_DIR \ + --suite=$STRESS_SUITE \ + --stress-tests-file=$STRESS_TEST_FILE \ + --stress-basedir=$STRESS_BASEDIR \ + --server-logs-dir=$STRESS_BASEDIR \ + --stress-mode=$STRESS_MODE \ + --mysqltest=$BASEDIR/client/mysqltest \ + --threads=$STRESS_THREADS \ + --verbose \ + --cleanup \ + --log-error-details \ + --abort-on-error" + + if [ -n "$STRESS_INIT_FILE" ] ; then + STRESS_TEST_ARGS="$STRESS_TEST_ARGS --stress-init-file=$STRESS_INIT_FILE" + fi + + if [ -n "$STRESS_LOOP_COUNT" ] ; then + STRESS_TEST_ARGS="$STRESS_TEST_ARGS --loop-count=$STRESS_LOOP_COUNT" + fi + + if [ -n "$STRESS_TEST_COUNT" ] ; then + STRESS_TEST_ARGS="$STRESS_TEST_ARGS --test-count=$STRESS_TEST_COUNT" + fi + + if [ -n "$STRESS_TEST_DURATION" ] ; then + STRESS_TEST_ARGS="$STRESS_TEST_ARGS --test-duration=$STRESS_TEST_DURATION" + fi + + echo "Stress test related variables:" + echo "TESTS - $1" + echo "STRESS - $DO_STRESS" + echo "STRESS_SUITE - $STRESS_SUITE" + echo "STRESS_TEST_FILE - $STRESS_TEST_FILE" + echo "STRESS_INIT_FILE - $STRESS_INIT_FILE" + echo "STRESS_THREADS - $STRESS_THREADS" + echo "STRESS_MODE - $STRESS_MODE" + echo "STRESS_TEST_COUNT - $STRESS_TEST_COUNT" + echo "STRESS_LOOP_COUNT - $STRESS_LOOP_COUNT" + echo "STRESS_TEST_DURATION - $STRESS_TEST_DURATION" + + #echo "$STRESS_TEST_ARGS"; + #Run stress test + $MYSQL_TEST_DIR/mysql-stress-test.pl $STRESS_TEST_ARGS + + +} + ###################################################################### # Main script starts here ###################################################################### @@ -1877,6 +2036,32 @@ then exit fi +# +# Stress testing +# +if [ "$DO_STRESS" = 1 ] +then + + if [ -n "$1" ] ; then + STRESS_TEST="$1"; + fi + + if [ $USE_RUNNING_SERVER -eq 0 ] ; then + start_master + fi + + run_stress_test + + if [ $USE_RUNNING_SERVER -eq 0 ] ; then + mysql_stop + stop_manager + fi + + exit + +fi + + $ECHO if [ x$USE_TIMER = x1 ] ; then $ECHO "TEST RESULT TIME (ms)"