mirror of
https://github.com/MariaDB/server.git
synced 2025-01-19 13:32:33 +01:00
277 lines
7.2 KiB
Text
277 lines
7.2 KiB
Text
#
|
|
# file : test/ndbnet/testError.run
|
|
# usage: perl testError.run
|
|
#
|
|
# you need to have $NDB_TOP/lib/perl5 on search path $PERL5LIB
|
|
# or else write perl -I$NDB_TOP/lib/perl5 test1.run
|
|
#
|
|
# The database is specified by the $NDB_DATABASE environment variable
|
|
#
|
|
# method names and argument style will change slightly.
|
|
#
|
|
#
|
|
|
|
use strict;
|
|
use NDB::Run;
|
|
|
|
my $env = NDB::Run->getenv;
|
|
my $log = $env->getlog;
|
|
$log->setpart(time => 1, line => 0);
|
|
$log->setprio("info");
|
|
|
|
my $database = $ENV{NDB_DATABASE};
|
|
my $api_cmd = $ENV{API_CMD};
|
|
my $api_cmd2 = $ENV{API_CMD2};
|
|
$log->put("start test database=$database");
|
|
$env->init or $log->push("init failed")->fatal;
|
|
|
|
my $db = $env->getdb($database) or $log->push->fatal;
|
|
my $mgm = $db->getnode(1) or $log->push->fatal;
|
|
my $api = $db->getnode(6) or $log->push->fatal;
|
|
my $api2 = $db->getnode(7) or $log->push->fatal;
|
|
|
|
my @dbnode = (); # array of db nodes indexed 2..5
|
|
for my $i (2..5) {
|
|
$dbnode[$i] = $db->getnode($i) or $log->push->fatal;
|
|
}
|
|
|
|
# list of db nodes and errors to insert
|
|
my @errors = ( # array of array refs
|
|
[ 3, 4, 8030, 8031 ],
|
|
[ 2, 5, 8027, 8030 ],
|
|
[ 2, 4, 5000, 5000 ],
|
|
[ 2, 5, 5000, 5000 ],
|
|
[ 3, 4, 5000, 7008 ],
|
|
[ 3, 5, 5000, 7008 ],
|
|
[ 2, 4, 5000, 7007 ],
|
|
[ 3, 4, 5000, 7007 ],
|
|
[ 3, 5, 5000, 7006 ],
|
|
[ 2, 5, 5000, 7006 ],
|
|
[ 2, 4, 5000, 7005 ],
|
|
[ 3, 5, 5000, 7005 ],
|
|
[ 3, 4, 7005, 7005 ],
|
|
[ 2, 5, 7005, 7005 ],
|
|
[ 3, 4, 7005, 7007 ],
|
|
[ 3, 5, 7005, 7007 ],
|
|
[ 2, 5, 7005, 7008 ],
|
|
[ 2, 4, 7005, 7008 ],
|
|
[ 3, 5, 7006, 7006 ],
|
|
[ 3, 4, 7006, 7006 ],
|
|
[ 2, 4, 7006, 7007 ],
|
|
[ 2, 5, 7006, 7007 ],
|
|
[ 2, 5, 7006, 7008 ],
|
|
[ 2, 4, 7006, 7008 ],
|
|
[ 3, 4, 7007, 7007 ],
|
|
[ 3, 5, 7007, 7007 ],
|
|
[ 3, 5, 7007, 7008 ],
|
|
[ 3, 4, 7007, 7008 ],
|
|
[ 2, 4, 7008, 7008 ],
|
|
[ 2, 5, 7008, 7008 ],
|
|
[ 2, 5, 7008, 7008 ],
|
|
[ 2, 5, 7008, 7008 ],
|
|
[ 2, 5, 8000, 8000 ],
|
|
[ 2, 4, 8000, 8000 ],
|
|
[ 3, 5, 8000, 8001 ],
|
|
[ 3, 4, 8000, 8001 ],
|
|
[ 2, 5, 8000, 5001 ],
|
|
[ 3, 5, 8000, 5001 ],
|
|
[ 3, 4, 8001, 5001 ],
|
|
[ 2, 4, 8001, 5001 ],
|
|
[ 2, 5, 8002, 8029 ],
|
|
[ 3, 4, 8030, 8031 ],
|
|
[ 3, 5, 8020, 8021 ],
|
|
[ 2, 4, 8022, 9999 ],
|
|
[ 2, 4, 8023, 8025 ],
|
|
[ 2, 5, 8027, 8030 ],
|
|
[ 3, 4, 8002, 8002 ],
|
|
[ 3, 5, 8029, 8030 ],
|
|
[ 3, 5, 8031, 8031 ],
|
|
[ 3, 4, 8020, 8020 ],
|
|
[ 2, 4, 8021, 8021 ],
|
|
[ 2, 5, 8022, 8022 ],
|
|
[ 3, 4, 8023, 8023 ],
|
|
[ 3, 5, 8025, 8025 ],
|
|
[ 2, 4, 8027, 8027 ],
|
|
[ 2, 5, 8027, 8027 ],
|
|
[ 3, 4, 8023, 9999 ],
|
|
[ 3, 5, 8025, 9998 ],
|
|
[ 2, 4, 8027, 9999 ],
|
|
[ 2, 5, 8027, 9998 ],
|
|
[ 3, 4, 8000, 9999 ],
|
|
[ 3, 5, 8001, 9998 ],
|
|
[ 2, 4, 5001, 9999 ],
|
|
[ 2, 5, 5000, 9999 ],
|
|
[ 3, 4, 7005, 9999 ],
|
|
[ 3, 5, 7006, 9999 ],
|
|
[ 2, 4, 7007, 9999 ],
|
|
[ 2, 5, 7008, 9999 ],
|
|
[ 2, 4, 9998, 9998 ],
|
|
[ 3, 5, 9998, 9998 ],
|
|
[ 2, 4, 9998, 9998 ],
|
|
[ 2, 4, 9998, 9998 ],
|
|
[ 3, 5, 9998, 9998 ],
|
|
[ 2, 4, 9999, 9999 ],
|
|
[ 2, 4, 9998, 9998 ],
|
|
[ 3, 5, 9999, 9999 ],
|
|
[ 2, 4, 9999, 9999 ],
|
|
[ 2, 4, 9999, 9999 ],
|
|
[ 3, 5, 9999, 9999 ],
|
|
);
|
|
|
|
$db->kill;
|
|
$db->start({init_rm=>1}) or $log->push->fatal;
|
|
sleep 10;
|
|
|
|
# should be option (or default) in $db->start
|
|
sub wait_until_started {
|
|
my $local_cnt = 60;
|
|
while (--$local_cnt > 0) {
|
|
sleep 5;
|
|
my $ret = $mgm->write("all status", { wait => 3 });
|
|
$ret or $log->fatal;
|
|
my $output = $ret->{output};
|
|
if ($output =~ /((.|\n)*\bstarted\b(.|\n)*){4}/i) {
|
|
$log->put("*** db is started ***")->info;
|
|
return;
|
|
}
|
|
if ($output =~ /((.|\n)*\bno.contact\b(.|\n)*){4}/i) {
|
|
print "NDBT_ProgramExit: 1 - test error\n";
|
|
$log->put("*** db is dead ***")->fatal;
|
|
}
|
|
}
|
|
print "NDBT_ProgramExit: 1 - test error\n";
|
|
$log->put("*** node recovery failed ***")->fatal;
|
|
}
|
|
|
|
sub getdbstatus {
|
|
my $ret = $mgm->write("all status", { wait => 3 });
|
|
$ret or return undef;
|
|
my $output = $ret->{output};
|
|
my @status = (); # indexed 2..5
|
|
for my $i (2..5) {
|
|
my $s = "Unknown";
|
|
if ($output =~ /Node\s*$i\s*:\s*(\w+)/i) {
|
|
$s = $1;
|
|
}#if
|
|
$status[$i] = lc $s;
|
|
}#for
|
|
return \@status;
|
|
}
|
|
|
|
# count elapsed time
|
|
my $lasttime;
|
|
sub settime { $lasttime = time };
|
|
sub gettime { return time - $lasttime };
|
|
|
|
wait_until_started();
|
|
$api->start({run=>$api_cmd});
|
|
sleep 15;
|
|
$api2->start({run=>$api_cmd2});
|
|
sleep 15;
|
|
|
|
# loop over error inserts
|
|
for my $x (0..1) {
|
|
for my $e (@errors) {
|
|
for my $z (0..3) {
|
|
my $i1 = $e->[0]; # db node number 2..5
|
|
my $i2 = $e->[1]; # db node number 2..5
|
|
my $c1 = $e->[2]; # error code
|
|
my $c2 = $e->[3]; # error code
|
|
my $dbnode1 = $dbnode[$i1];
|
|
my $dbnode2 = $dbnode[$i2];
|
|
my $kill_no = 9998;
|
|
my @survivor = ();
|
|
my $survivor_count = 0;
|
|
|
|
$log->put("insert error $c1 on node $i1")->push($dbnode1)->notice;
|
|
$log->put("and insert error $c2 on node $i2")->push($dbnode2)->notice;
|
|
|
|
for my $node (2..5) {
|
|
if (($node ne $i1) && ($node ne $i2)) {
|
|
$survivor[$survivor_count] = $node;
|
|
$survivor_count++;
|
|
}#if
|
|
}#for
|
|
# insert error
|
|
if ($c1 eq $kill_no) {
|
|
$dbnode1->kill
|
|
or $log->put("Kill 1 failed")->fatal;
|
|
} else {
|
|
$mgm->write("$i1 error $c1")
|
|
or $log->put("insert error 1 fault")->fatal;
|
|
}#if
|
|
if ($c2 eq $kill_no) {
|
|
$dbnode2->kill
|
|
or $log->put("Kill 2 failed")->fatal;
|
|
} else {
|
|
$mgm->write("$i2 error $c2")
|
|
or $log->put("insert error 2 fault")->fatal;
|
|
}#if
|
|
|
|
# after a few seconds check that node is dead
|
|
settime();
|
|
loop: {
|
|
gettime() <= 300
|
|
or $log->put("db node $i1 or $i2 refuses to die")->fatal;
|
|
my $status = getdbstatus()
|
|
or $log->put("getdbstatus error")->fatal;
|
|
|
|
if (($status->[$survivor[0]] eq 'no') ||
|
|
($status->[$survivor[1]] eq 'no')) {
|
|
print "NDBT_ProgramExit: 1 - test error\n";
|
|
$db->kill;
|
|
$log->put("*** db is dead ***")->fatal;
|
|
}#if
|
|
if (($status->[$i1] ne 'no') || ($status->[$i2] ne 'no')) { # ...contact
|
|
sleep 2;
|
|
redo loop;
|
|
}#if
|
|
}#loop
|
|
|
|
# have to even check the process is gone
|
|
sleep 5;
|
|
if ($dbnode1->stat ne "down") {
|
|
$log->put("ndb did not die, kill it")->push($dbnode1)->warn;
|
|
$dbnode1->kill
|
|
or $log->put("Kill 3 failed")->fatal;
|
|
}#if
|
|
if ($dbnode2->stat ne "down") {
|
|
$log->put("ndb did not die, kill it")->push($dbnode2)->warn;
|
|
$dbnode2->kill
|
|
or $log->put("Kill 4 failed")->fatal;
|
|
}#if
|
|
|
|
$log->put("node $i1 and node $i2 is dead")->notice;
|
|
|
|
# start the failed nodes
|
|
$dbnode1->start
|
|
or $log->put("Start ndb node 1 failed")->fatal;
|
|
$dbnode2->start
|
|
or $log->put("Start ndb node 2 failed")->fatal;
|
|
wait_until_started();
|
|
$log->put("node $i1 and node $i2 is up again")->notice;
|
|
|
|
# check test pgm is running
|
|
my $stat = $api->stat
|
|
or $log->put("api->stat failed")->fatal;
|
|
if ($stat ne "up") {
|
|
print "NDBT_ProgramExit: 1 - test error\n";
|
|
sleep 15;
|
|
$db->kill;
|
|
$log->put("flexBench has crashed")->fatal;
|
|
}#if
|
|
my $stat = $api2->stat
|
|
or $log->put("api2->stat failed")->fatal;
|
|
if ($stat ne "up") {
|
|
print "NDBT_ProgramExit: 1 - test error\n";
|
|
sleep 15;
|
|
$db->kill;
|
|
$log->put("flexBench2 has crashed")->fatal;
|
|
}#if
|
|
}#for
|
|
}#for
|
|
}#for
|
|
print "NDBT_ProgramExit: 0 - test ready\n";
|
|
$db->kill;
|
|
|
|
# vim: set sw=4:
|