#
# file : test/ndbnet/testError.run
# usage: perl testError.run
#
# you need to have $NDB_TOP/lib/perl5 on search path $PERL5LIB
# or else write perl -I$NDB_TOP/lib/perl5 test1.run
#
# The database is specified by the $NDB_DATABASE environment variable
#
# method names and argument style will change slightly.
#
#

use strict;
use NDB::Run;

my $env = NDB::Run->getenv;
my $log = $env->getlog;
$log->setpart(time => 1, line => 0);
$log->setprio("info");

my $database = $ENV{NDB_DATABASE};
my $api_cmd = $ENV{API_CMD};
$log->put("start test database=$database");
$env->init or $log->push("init failed")->fatal;

my $db = $env->getdb($database) or $log->push->fatal;
my $mgm = $db->getnode(1) or $log->push->fatal;
my $api = $db->getnode(4) or $log->push->fatal;

my @dbnode = ();	# array of db nodes indexed 2..3
for my $i (2..3) {
    $dbnode[$i] = $db->getnode($i) or $log->push->fatal;
}

# list of db nodes and errors to insert
my @errors = (		# array of array refs
    [ 2, 9998 ],
    [ 2, 37017 ],
    [ 2, 37018 ],
    [ 2, 37017 ],
    [ 2, 37018 ],
    [ 2, 37017 ],
    [ 2, 37018 ],
    [ 2, 37017 ],
    [ 2, 37018 ],
    [ 2, 37017 ],
    [ 2, 37018 ],
    [ 2, 37017 ],
    [ 2, 9998 ],
    [ 3, 9998 ],
    [ 3, 9998 ],
    [ 2, 9998 ],
    [ 3, 9998 ],
    [ 3, 9998 ],
    [ 2, 38002 ],
    [ 2, 38002 ],
    [ 2, 8002 ],
    [ 3, 8029 ],
    [ 2, 8030 ],
    [ 2, 8031 ],
    [ 3, 8020 ],
    [ 2, 8021 ],
    [ 3, 8022 ],
    [ 2, 8023 ],
    [ 3, 8025 ],
    [ 2, 8027 ],
    [ 2, 38002 ],
    [ 3, 38029 ],
    [ 2, 38030 ],
    [ 2, 38031 ],
    [ 3, 38020 ],
    [ 2, 38021 ],
    [ 3, 38022 ],
    [ 2, 38023 ],
    [ 3, 38025 ],
    [ 2, 38027 ],
    [ 2, 48002 ],
    [ 3, 48029 ],
    [ 2, 48030 ],
    [ 2, 48031 ],
    [ 3, 48020 ],
    [ 2, 48021 ],
    [ 3, 48022 ],
    [ 2, 48023 ],
    [ 3, 48025 ],
    [ 2, 48027 ],
    [ 2, 9999 ],
    [ 3, 9999 ],
    [ 3, 9999 ],
    [ 2, 9998 ],
    [ 3, 9998 ],
    [ 3, 9998 ],
    [ 2, 9998 ],
    [ 3, 9998 ],
    [ 3, 9998 ],
    [ 3, 37000 ],
    [ 2, 37001 ],
    [ 2, 37002 ],
    [ 2, 37003 ],
    [ 2, 47005 ],
    [ 2, 47006 ],
    [ 2, 47007 ],
    [ 2, 47008 ],
    [ 2, 45000 ],
    [ 2, 37005 ],
    [ 2, 37006 ],
    [ 2, 37007 ],
    [ 2, 37008 ],
    [ 2, 35000 ],
    [ 2, 37009 ],
    [ 2, 37010 ],
    [ 2, 37013 ],
    [ 2, 37014 ],
    [ 2, 37015 ],
    [ 2, 37016 ],
    [ 2, 37017 ],
    [ 2, 37018 ],
    [ 2, 37019 ],
    [ 2, 47020 ],
    [ 2, 37020 ],
    [ 2, 48000 ],
    [ 2, 38000 ],
    [ 2, 48001 ],
    [ 2, 38001 ],
    [ 2, 45001 ],
    [ 2, 35001 ],
);

$db->kill;
$db->start({init_rm=>1}) or $log->push->fatal;
sleep 10;

# should be option (or default) in $db->start
sub wait_until_started {
    my $local_cnt = 60;
    while (--$local_cnt > 0) {
	sleep 2;
	my $ret = $mgm->write("all status", { wait => 3 });
	$ret or $log->fatal;
	my $output = $ret->{output};
	if ($output =~ /\bstarted\b(.|\n)*started\b/i) {
	    $log->put("*** db is started ***")->info;
	    return;
	}
	if ($output =~ /\bno.contact\b(.|\n)*no.contact\b/i) {
	    print "NDBT_ProgramExit: 1 - test error\n";
	    $log->put("*** db is dead ***")->fatal;
	}
    }
    print "NDBT_ProgramExit: 1 - test error\n";
    $log->put("*** node recovery failed ***")->fatal;
}

sub getdbstatus {
    my $ret = $mgm->write("all status", { wait => 3 });
    $ret or return undef;
    my $output = $ret->{output};
    my @status = ();	# indexed 2..3
    for my $i (2..3) {
	my $s = "Unknown";
	if ($output =~ /Node\s*$i\s*:\s*(\w+)/i) {
	    $s = $1;
	}#if
	$status[$i] = lc $s;
    }#for
    return \@status;
}

# count elapsed time
my $lasttime;
sub settime { $lasttime = time };
sub gettime { return time - $lasttime };

wait_until_started();
$api->start({run=>$api_cmd});
sleep 10;

# loop over error inserts
for my $e (@errors) {
for my $loop (1..7) {
  my $i = $e->[0];                      # db node number 2..3
  my $c = $e->[1];                      # error code
  my $dead_node_id = $i;
  my $dbnode = $dbnode[$i];
  my $two = 2;
  my $three = 3;
  my $kill_no = 9998;

  $log->put("insert error $c")->push($dbnode)->notice;

  # insert error
  if ($c eq $kill_no) {
    $dbnode->kill
       or $log->put("Kill 1 failed")->fatal;
  } else {
    $mgm->write("$i error $c")
       or $log->put("insert error fault")->fatal;
  }#if

  # after a few seconds check that node is dead
  settime();
  loop: {
    gettime() <= 300
      or $log->put("db node $i refuses to die")->fatal;
    my $status = getdbstatus()
       or $log->put("getdbstatus error")->fatal;

    if (($status->[$two] eq 'no') && ($status->[$three] eq 'no')) {
      print "NDBT_ProgramExit: 1 - test error\n";
      $log->put("*** db is dead ***")->fatal;
    }#if
    if ($c < 10000) {
      if ($status->[$i] ne 'no') {         # ...contact
        sleep 2;
        redo loop;
      }#if
      $dead_node_id = $i;
    } else {
      if (($status->[$two] eq 'no') &&
          ($status->[$three] eq 'started')) {
        $dead_node_id = 2;
      } else {
        if (($status->[$three] eq 'no') &&
            ($status->[$two] eq 'started')) {
          $dead_node_id = 3;
        } else {
          sleep 2;
          redo loop;
        }#if
      }#if
    }#if
  }#loop

  my $dead_node = $dbnode[$dead_node_id];
  # have to even check the process is gone
  sleep 5;
  if ($dead_node->stat ne "down") {
    $log->put("ndb did not die, kill it")->push($dead_node)->warn;
    $dead_node->kill
       or $log->put("Kill 2 failed")->fatal;
  }#if

  $log->put("node $dead_node_id is dead")->notice;

  # start the failed node
  $dead_node->start
     or $log->put("Start ndb node failed")->fatal;

  wait_until_started();
  $log->put("node $dead_node_id is up again")->notice;

  # check test pgm is running
  my $stat = $api->stat
    or $log->put("api->stat failed")->fatal;
  if ($stat ne "up") {
    print "NDBT_ProgramExit: 1 - test error\n";
    $db->kill;
    $log->put("flexBench has crashed")->fatal;
  }#if
}#for
}#for
print "NDBT_ProgramExit: 0 - test ready\n";
$db->kill;

# vim: set sw=4: