mariadb/Build-tools/Do-compile

#!/usr/bin/perl

use Getopt::Long;
$opt_distribution=$opt_user=$opt_result=$opt_config_options=$opt_config_env="";
$opt_dbd_options=$opt_perl_options=$opt_suffix="";
$opt_tmp=$version_suffix="";
$opt_help=$opt_Information=$opt_no_delete=$opt_debug=$opt_stage=$opt_rsh_mail=$opt_no_test=$opt_no_perl=$opt_with_low_memory=$opt_fast_benchmark=$opt_static_client=$opt_static_server=$opt_static_perl=$opt_sur=$opt_with_small_disk=$opt_local_perl=$opt_tcpip=$opt_build_thread=$opt_no_mysqltest=$opt_use_old_distribution=$opt_enable_shared=$opt_no_crash_me=$opt_no_strip=0;
$opt_innodb=$opt_bdb=0;

GetOptions("Information","help","distribution=s","user=s","result=s","no-delete","no-test","no-mysqltest","perl-files=s","debug","config-options=s","config-env=s","stage=i","rsh-mail","with-low-memory","fast-benchmark","tmp=s","static-client","static-server","static-perl","no-perl","local-perl","perl-options=s","sur","with-small-disk","dbd-options=s","tcpip","suffix=s","build-thread=i","innodb","bdb","use-old-distribution","enable-shared","no-crash-me","no-strip") || usage();

usage() if ($opt_help || $opt_Information);
usage() if (!$opt_distribution);

if ($opt_innodb || $opt_bdb)
{
  $version_suffix="-max";
}

chomp($host=`hostname`);
$full_host_name=$host;
info("Compiling MySQL$version_suffix at $host$suffix, stage: $opt_stage\n");
$connect_option= ($opt_tcpip ? "--host=$host" : "");
$host =~ /^([^.-]*)/;
$host=$1 . $opt_suffix;
$email="$opt_user\@mysql.com";
$pwd = `pwd`; chomp($pwd);
$log="$pwd/Logs/$host$version_suffix.log";
$opt_distribution =~ /(mysql-[^\/]*)\.tar/;
$ver=$1;
$gcc_version=which("gcc");
if (defined($gcc_version) && ! $opt_config_env)
{
  $tmp=`$gcc_version -v 2>&1`;
  if ($tmp =~ /version 2\.7\./)
  {
    $opt_config_env= 'CC=gcc CFLAGS="-O2 -fno-omit-frame-pointer" CXX=gcc CXXFLAGS="-O2 -fno-omit-frame-pointer"';
  }
  else
  {
    $opt_config_env= 'CC=gcc CFLAGS="-O3 -fno-omit-frame-pointer" CXX=gcc CXXFLAGS="-O3 -fno-omit-frame-pointer -felide-constructors -fno-exceptions -fno-rtti"';
  }
}

$new_opt_tmp=0;
if ($opt_tmp)
{
  if (! -d $opt_tmp)
  {
    safe_system("mkdir $opt_tmp");
    $new_opt_tmp=1;
  }
  $ENV{'TMPDIR'}=$opt_tmp;
}
else
{
  $opt_tmp="/tmp";
}
$bench_tmpdir="$opt_tmp/my_build-$host";
$ENV{'PATH'}= "$pwd/$host/bin:" . $ENV{'PATH'}; 
$rsh=which("rcmd","rsh");
$make=which("gmake","make");	# Can't use -j here!
$tar=which("gtar","tar");
$sendmail=find("/usr/lib/sendmail","/usr/sbin/sendmail");
$sur= $opt_sur ? "/my/local/bin/sur" : "";
delete $ENV{'MYSQL_PWD'};	# Reset possibly password
delete $ENV{'MY_BASEDIR_VERSION'};
$ENV{'MYSQL_TCP_PORT'}= $mysql_tcp_port= 3334 + $opt_build_thread;
$ENV{'MYSQL_UNIX_PORT'}=$mysql_unix_port="$opt_tmp/mysql$opt_suffix.build";
$ENV{"PERL5LIB"}="$pwd/$host/perl5:$pwd/$host/perl5/site_perl";

if ($opt_stage == 0)
{
  print "$host: Removing old distribution\n" if ($opt_debug);
  if (-x "$host/bin/mysqladmin")
  {
    log_system("$host/bin/mysqladmin --no-defaults -u root -S $mysql_unix_port -s shutdown");
    log_system("$host/bin/mysqladmin --no-defaults -u root -P $mysql_tcp_port -h $host -s shutdown");
    log_system("$host/bin/mysqladmin --no-defaults -u root -P 9306 -h $host -s shutdown");
    log_system("$host/bin/mysqladmin --no-defaults -u root -P 9307 -h $host -s shutdown");
  }
  if (!$opt_use_old_distribution)
  {
    system("mkdir $host") if (! -d $host);
    system("touch $host/mysql-fix-for-glob");
    rm_all(<$host/mysql-*>);
    system("mkdir $host/bin") if (! -d "$host/bin");
  }
  system("mkdir $bench_tmpdir") if (! -d $bench_tmpdir);
  rm_all("$host/test");
  system("mkdir $host/test") if (! -d "$host/test");
  system("mkdir Logs") if (! -d "Logs");
  system("mv $log ${log}-old") if (-f $log);
  unlink($log);
}

open(LOG,">>$log") || abort("Can't open log file, error $?");
select LOG;
$|=1;
select STDOUT;
$|=1;

safe_cd("$host");
if ($opt_stage == 0 && ! $opt_use_old_distribution)
{
  safe_system("gunzip < $opt_distribution | $tar xf -");
}
safe_cd($ver);
if ($opt_stage <= 1)
{
  $opt_config_options.=" --with-low-memory" if ($opt_with_low_memory);
  unlink("config.cache");
  log_system("$make clean") if ($opt_use_old_distribution);
  if ($opt_static_server)
  {
    $opt_config_options.=" --with-mysqld-ldflags=-all-static";
  }
  if ($opt_static_client)
  {
    $opt_config_options.=" --with-client-ldflags=-all-static";
  }
  if (!$opt_enable_shared)
  {
    $opt_config_options.= " --disable-shared"; # Default for binary versions
  }
  if ($opt_bdb)
  {
    $opt_config_options.= " --with-berkeley-db"
  }
  if ($opt_innodb)
  {
    $opt_config_options.= " --with-innodb"
  }
  check_system("$opt_config_env ./configure --prefix=/usr/local/mysql \"--with-comment=Official MySQL$version_suffix binary\" --with-extra-charsets=complex \"--with-server-suffix=$version_suffix\" $opt_config_options","Thank you for choosing MySQL");
  if (-d "$pwd/$host/include-mysql")
  {
    safe_system("cp -r $pwd/$host/include-mysql/* $pwd/$host/$ver/include");
  }
}

if ($opt_stage <= 2)
{
  unlink($opt_distribution) if (!$opt_no_delete && !$opt_use_old_distribution);
  safe_system("$make");
}

#
# Create a binary distrubtion
#
if ($opt_stage <= 3)
{
  my ($flags);
  log_system("rm -fr mysql-3* mysql-4* $pwd/$host/*.tar.gz");
  log_system("nm -n sql/mysqld | gzip -9 -v 2>&1 > sql/mysqld.sym.gz | cat");

  $flags="";
  $flags.="--no-strip" if ($opt_no_strip);
  check_system("scripts/make_binary_distribution --tmp=$opt_tmp --suffix=$opt_suffix $flags",".tar.gz created");
  safe_system("mv mysql*.tar.gz $pwd/$host");
  safe_system("cp client/mysqladmin $pwd/$host/bin");
  safe_system("$make clean") if ($opt_with_small_disk);
}

#
# Unpack and test the binary distrubtion
#

$tar_file=<$pwd/$host/*.tar.gz>;
if (!defined($tar_file))
{
  $tar_file=<$pwd/$host/*.tgz>;
}
if ($opt_stage <= 4 && !$opt_no_test)
{
  rm_all(<$pwd/$host/test/*>);
  safe_cd("$pwd/$host/test");
  safe_system("gunzip < $tar_file | $tar xf -");
}

$tar_file =~ /(mysql-[^\/]*)\.tar/;
$ver=$1;
$test_dir="$pwd/$host/test/$ver";
$ENV{"LD_LIBRARY_PATH"}= "$testdir/lib:" . $ENV{"LD_LIBRARY_PATH"};

if ($opt_stage <= 5 && !$opt_no_test && !$opt_no_mysqltest)
{
  my ($slave_port);
  $slave_port=$mysql_tcp_port+16;
  safe_cd("${test_dir}/mysql-test");
  check_system("./mysql-test-run --tmpdir=$bench_tmpdir --master_port=$mysql_tcp_port --slave_port=$slave_port --sleep=10", "tests were successful");
}

if ($opt_stage <= 6 && !$opt_no_test)
{
  my $extra;
  safe_cd($test_dir);
  log_system("./bin/mysqladmin --no-defaults -u root -S $mysql_unix_port -s shutdown") || info("There was no mysqld running\n");
  sleep(2);
  log_system("rm -f  ./data/mysql/*");
  check_system("scripts/mysql_install_db --no-defaults --skip-locking","https://order");
  $extra="";
  if ($opt_bdb)
  {
    $extra.=" -O bdb_cache_size=16M";
  }
  if ($opt_innodb)
  {
    $extra.=" --innodb_data_file_path=ibdata1:100M";
  }
  safe_system("./bin/mysqld --no-defaults --basedir . --datadir ./data --skip-locking $extra >> $log 2>&1 &");
  sleep(2);
}

if ($opt_stage <= 7 && $opt_perl_files && !$opt_no_perl && !$opt_no_test)
{
  safe_cd($test_dir);
  rm_all("perl");
  safe_system("mkdir perl");
  $ENV{'IN_MYSQL_DISTRIBUTION'}=1;
  $ENV{'MYSQL_BUILD'}=$test_dir;

  foreach $module (split(/,/,$opt_perl_files))
  {
    my $options;
    safe_cd("$test_dir/perl");
    if ($opt_debug)
    {
      safe_system("gunzip < $pwd/$module | tar xvf -");
    }
    else
    {
      safe_system("gunzip < $pwd/$module | tar xf -");
    }
    $module =~ m|([^/]+)\.tar\.gz|;
    $module = $1;
    safe_cd($module);
    $options="";
    $options= "--mysql-install --noprompt --mysql-incdir=$test_dir/include --mysql-libdir=$test_dir/lib -nomsql-install -nomsql1-install --mysql-test-db=test $opt_dbd_options" if ($module =~ /Msql-Mysql/);
    $options.= " PREFIX=$pwd/$host INSTALLPRIVLIB=$pwd/$host/perl5 INSTALLSCRIPT=$pwd/$host/bin INSTALLSITELIB=$pwd/$host/perl5/site_perl INSTALLBIN=$pwd/$host/bin INSTALLMAN1DIR=$pwd/$host/man INSTALLMAN3DIR=$pwd/$host/man/man3" if ($opt_local_perl);
    $options.= " $opt_perl_options" if (defined($opt_perl_options));
    safe_system($opt_static_perl ? "perl Makefile.PL -static $options" : "perl Makefile.PL $options");
    safe_system("$make ; $sur $make install");
  }
}


if ($opt_stage <= 8 && !$opt_no_test && !$opt_no_crash_me)
{
  safe_cd("$test_dir/sql-bench");
  log_system("rm -f limits/mysql.cfg");
  safe_system("perl ./crash-me --force --batch-mode $connect_option");
}

if ($opt_stage <= 9 && !$opt_no_test)
{
  safe_cd("$test_dir/sql-bench");
  log_system("rm -f output/*");
  $tmp= $opt_fast_benchmark ? "--fast --user root --small-test" : "";
  check_system("perl ./run-all-tests --log --die-on-errors $connect_option $tmp","RUN-mysql");
  if ($opt_bdb)
  {
    check_system("perl ./run-all-tests --log --suffix=\"_bdb\" --die-on-errors $connect_option $tmp --create-option=\"type=bdb\"","RUN-mysql");
  }
  if ($opt_innodb)
  {
    check_system("perl ./run-all-tests --log --suffix=\"_innodb\" --die-on-errors $connect_option $tmp --create-option=\"type=innodb\"","RUN-mysql");
  }
}

if ($opt_stage <= 10 && $opt_result)
{
  safe_system("rcp $pwd/$host/*.tar.gz $opt_result");
}

rm_all($bench_tmpdir);
rm_all("$opt_tmp") if ($new_opt_tmp);

log_system("$pwd/$host/bin/mysqladmin --no-defaults -S $mysql_unix_port -u root shutdown");
print LOG "ok\n";
close LOG;
print "$host: ok\n";

exit 0;


sub usage
{
print <<EOF;
$0  version 1.2

$0 takes the following options:

--help or --Information
Show this help

--innodb
Compile with support for Innodb tables

--bdb
Compile with support for Berkeley DB tables

--user 'user_name'
Mail 'user_name'\@analytikerna.se if something went wrong.
If user is empty then no mail is sent.

--distribution 'distribution_file'
Name of the MySQL distribution file.

--result 'result_dir'
Send binary distribution to result_dir

--no-delete
Do not delete the distribution file.

--perl-files=list of files
Compile and install the given perl modules.

--no-test
Don not run crash-me or the benchmarks.

--config-options 'some extra configure options'
To add some extra options, like '--with-perl=yes'

--config-env 'environment for configure'
To set up the environment, like 'CC=cc CXX=gcc CXXFLAGS=-O3'

--dbd-options 'options'
Options for Makefile.PL when configuring msql-mysql-modules.

--with-low-memory
Use less memory when compiling.

-- stage (1-6)
Start script from some specific point.

-- debug
Print all shell commands on stdout.
EOF
exit 1;
} 

sub abort
{
  my($message)=@_;
  my($command,$mail_header_file);
  print LOG "\n$message\n";
  print "$host: $message\n" if ($opt_debug);
  close LOG;

  if ($opt_user)
  {
    $mail_header_file="$opt_tmp/do-command.$$";
    open(TMP,">$mail_header_file");
    print TMP "From: mysql\@$full_host_name\n";
    print TMP "To: $email\n";
    print TMP "Subject: MySQL compilation failed again\n\n";
    close TMP;
    $command=($opt_rsh_mail) ? "$rsh analytik" : "";
    system("tail -30 $log > $log.mail");
    system("cat $mail_header_file $log.mail | $command $sendmail -t -f $email");
    unlink($mail_header_file);
    unlink("$log.mail");
  }
  print LOG "Aborting\n";
  exit 1;
}

sub info
{
  my($message)=@_;
  print LOG "$message\n";
  print "$host: $message\n";
}

sub log_system
{
  my($com)=@_;
  print "$host: $com\n" if ($opt_debug);
  if (defined($log))
  {
    print LOG "$com\n";
    system("$com >> $log 2>&1") &&
      print LOG ("Info: couldn't execute command, error: " . ($? / 256) ."\n");
  }
  else
  {
    system($com) && print "$host: Couldn't execute command, error: " . ($? / 256) ."\n";
  }
}


sub safe_system
{
  my($com,$res)=@_;
  print LOG "$com\n";
  print "$host: $com\n" if ($opt_debug);
  system("$com >> $log 2>&1") && abort("error: Couldn't execute command, error: " . ($? / 256));
}

sub check_system
{
  my($com,$res)=@_;
  my ($error,$found);
  print LOG "$com\n";
  print "$host: $com\n" if ($opt_debug);
  open (COM, "$com 2>&1 < /dev/null|") || abort("Got error " . ($?/256) ." opening pipe");
  $found=0;
  while (<COM>)
  {
    print LOG $_;
    if (index($_,$res) >= 0)
    {
      $found=1;
      last;
    }
  }
  close COM;
  abort("Couldn't find '$res' in the command result") if (!$found);
  print "$host: Command ok\n" if ($opt_debug);
}

sub safe_cd
{
  my($dir)=@_;
  print LOG "cd $dir\n";
  print "$host: cd $dir\n" if ($opt_debug);
  chdir($dir) || abort("Can't cd to $dir");
}

sub which
{
  my(@progs)=@_;
  foreach $prog (@progs)
  {
    chomp($found=`which $prog | head -1`);
    if ($? == 0 && $found ne "" && index($found," ") == -1)
    {
      $found =~ s|/+|/|g;	# Make nicer output
      return $found;
    }
  }
  return undef();
}

sub find
{
  my (@progs)=@_;
  foreach $prog (@progs)
  {
    return $prog if (-x $prog);
  }
  return undef();  
}

#
# Remove recursively all from a directory
# This is needed because problems with NFS and open files
#

sub rm_all
{
  my(@rm_files)=@_;
  my($dir,$current_dir,@files,@dirs);
  $current_dir = `pwd`; chomp($current_dir);

  foreach $dir (@rm_files)
  {
    if (-d $dir)
    {
      chdir($dir) || abort("Can't cd to $dir");
      print "$host: Removing from $dir\n" if ($opt_debug);
      while (<* .*>)
      {
	next if ($_ eq "." x (length($_)));
	if (-d $_)
	{
#	  die "Can't remove directory that starts with ." if ($_ =~ /^\./ && $_ ne ".libs"); # Safety
	  push (@dirs,$_);
	}
	else
	{
	  push (@files,$_);
	}
      }
      if ($#files >= 0)
      {
	system("rm -f " . join(" ",@files)) && abort("Can't remove files from $dir");
      }
      foreach $dir (@dirs)
      {
	rm_all($dir);
      }
      chdir($current_dir) || abort("Can't cd to $current_dir");
      log_system("rmdir $dir");
    }
    else
    {
      system("rm -f $dir") && abort("Can't remove file $dir");
    }
  }
}