Merge mysql.com:/users/lthalmann/bkroot/mysql-5.0

into  mysql.com:/users/lthalmann/bk/mysql-5.0-mysqlbinlog-ulong

BitKeeper/etc/config

@@ -73,5 +73,6 @@ hours:
 [nick:]checkout:get
 [jonas:]checkout:get
 [tomas:]checkout:get
+[guilhem:]checkout:get
 checkout:edit
 eoln:unix

include/my_sys.h

@@ -158,7 +158,8 @@ extern gptr my_memdup(const byte *from,uint length,myf MyFlags);
 extern char *my_strdup(const char *from,myf MyFlags);
 extern char *my_strdup_with_length(const byte *from, uint length,
 				   myf MyFlags);
-#define my_free(PTR,FG) my_no_flags_free(PTR)
+/* we do use FG (as a no-op) in below so that a typo on FG is caught */
+#define my_free(PTR,FG) ((void)FG,my_no_flags_free(PTR))
 #define CALLER_INFO_PROTO   /* nothing */
 #define CALLER_INFO         /* nothing */
 #define ORIG_CALLER_INFO    /* nothing */

mysql-test/r/federated.result

@@ -79,7 +79,7 @@ Table	Create Table
 t2	CREATE TABLE `t2` (
   `id` int(20) NOT NULL,
   `name` varchar(32) NOT NULL default ''
-) ENGINE=FEDERATED DEFAULT CHARSET=latin1 CONNECTION='mysql://root@127.0.0.1:9308/federated/t1'
+) ENGINE=FEDERATED DEFAULT CHARSET=latin1 CONNECTION='mysql://root@127.0.0.1:SLAVE_PORT/federated/t1'
 INSERT INTO federated.t2 (id, name) VALUES (1, 'foo');
 INSERT INTO federated.t2 (id, name) VALUES (2, 'fee');
 SELECT * FROM federated.t2;

mysql-test/r/rpl_dual_pos_advance.result

@@ -0,0 +1,22 @@
+stop slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+reset master;
+reset slave;
+drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9;
+start slave;
+reset master;
+change master to master_host="127.0.0.1",master_port=SLAVE_PORT,master_user="root";
+start slave;
+create table t1 (n int);
+create table t4 (n int);
+create table t5 (n int);
+create table t6 (n int);
+show tables;
+Tables_in_test
+t1
+t4
+t5
+t6
+stop slave;
+reset slave;
+drop table t1,t4,t5,t6;

mysql-test/r/subselect.result

@@ -2987,3 +2987,4 @@ select * from (select max(fld) from t1) as foo;
 max(fld)
 1
 drop table t1;
+purge master logs before (select adddate(current_timestamp(), interval -4 day));

mysql-test/t/federated.test

@@ -75,7 +75,8 @@ eval CREATE TABLE federated.t2 (
   ENGINE="FEDERATED" DEFAULT CHARSET=latin1
   CONNECTION='mysql://root@127.0.0.1:$SLAVE_MYPORT/federated/t1';
 
-SHOW CREATE TABLE federated.t2;
+--replace_result $SLAVE_MYPORT SLAVE_PORT
+eval SHOW CREATE TABLE federated.t2;
 
 INSERT INTO federated.t2 (id, name) VALUES (1, 'foo');
 INSERT INTO federated.t2 (id, name) VALUES (2, 'fee');

mysql-test/t/rpl_dual_pos_advance-master.opt

@@ -0,0 +1 @@
+--loose-to-force-a-restart

mysql-test/t/rpl_dual_pos_advance.test

@@ -0,0 +1,108 @@
+# This test checks that in a dual-head setup
+# A->B->A, where A has --log-slave-updates (why would it?
+# assume that there is a C as slave of A),
+# then the Exec_master_log_pos of SHOW SLAVE STATUS does
+# not stay too low on B(BUG#13023 due to events ignored because
+# of their server id).
+# It also will test BUG#13861.
+
+source include/master-slave.inc;
+
+
+# set up "dual head"
+
+connection slave;
+reset master;
+
+connection master;
+--replace_result $SLAVE_MYPORT SLAVE_PORT
+eval change master to master_host="127.0.0.1",master_port=$SLAVE_MYPORT,master_user="root";
+
+start slave;
+
+# now we test it
+
+connection slave;
+
+create table t1 (n int);
+
+save_master_pos;
+connection master;
+sync_with_master;
+
+# Now test BUG#13861. This will be enabled when Guilhem fixes this
+# bug.
+
+# stop slave
+
+# create table t2 (n int); # create one ignored event
+
+# save_master_pos;
+# connection slave;
+# sync_with_master;
+
+# connection slave;
+
+# show tables;
+
+# save_master_pos;
+
+# create table t3 (n int);
+
+# connection master;
+
+# bug is that START SLAVE UNTIL may stop too late, we test that by
+# asking it to stop before creation of t3.
+
+# start slave until master_log_file="slave-bin.000001",master_log_pos=195;
+
+# wait until it's started (the position below is the start of "CREATE
+# TABLE t2") (otherwise wait_for_slave_to_stop may return at once)
+
+# select master_pos_wait("slave-bin.000001",137);
+
+# wait_for_slave_to_stop;
+
+# then BUG#13861 causes t3 to show up below (because stopped too
+# late).
+
+# show tables;
+
+# start slave;
+
+# BUG#13023 is that Exec_master_log_pos may stay too low "forever":
+
+connection master;
+
+create table t4 (n int); # create 3 ignored events
+create table t5 (n int);
+create table t6 (n int);
+
+save_master_pos;
+connection slave;
+sync_with_master;
+
+connection slave;
+
+save_master_pos;
+
+connection master;
+
+# then BUG#13023 caused hang below ("master" looks behind, while it's
+# not in terms of updates done).
+
+sync_with_master;
+
+show tables;
+
+# cleanup
+
+stop slave;
+reset slave;
+drop table t1,t4,t5,t6; # add t2 and t3 later
+
+save_master_pos;
+connection slave;
+sync_with_master;
+
+# End of 4.1 tests

mysql-test/t/subselect.test

@@ -1962,4 +1962,9 @@ insert into t1 values ('1');
 select * from (select max(fld) from t1) as foo;
 drop table t1;
 
+#
+# BUG #10308: purge log with subselect
+#
+
+purge master logs before (select adddate(current_timestamp(), interval -4 day));
 

mysys/my_realloc.c

@@ -52,7 +52,7 @@ gptr my_realloc(gptr oldpoint, uint size, myf my_flags)
   if ((point = (char*)realloc(oldpoint,size)) == NULL)
   {
     if (my_flags & MY_FREE_ON_ERROR)
-      my_free(oldpoint,MyFLAGS);
+      my_free(oldpoint, my_flags);
     if (my_flags & MY_HOLD_ON_ERROR)
       DBUG_RETURN(oldpoint);
     my_errno=errno;

sql/log.cc

@@ -1353,7 +1353,8 @@ void MYSQL_LOG::new_file(bool need_lock)
         to change base names at some point.
       */
       THD *thd = current_thd; /* may be 0 if we are reacting to SIGHUP */
-      Rotate_log_event r(thd,new_name+dirname_length(new_name));
+      Rotate_log_event r(thd,new_name+dirname_length(new_name),
+                         0, LOG_EVENT_OFFSET, 0);
       r.write(&log_file);
       bytes_written += r.data_written;
     }
@@ -1432,7 +1433,7 @@ bool MYSQL_LOG::appendv(const char* buf, uint len,...)
 
   DBUG_ASSERT(log_file.type == SEQ_READ_APPEND);
 
-  pthread_mutex_lock(&LOCK_log);
+  safe_mutex_assert_owner(&LOCK_log);
   do
   {
     if (my_b_append(&log_file,(byte*) buf,len))
@@ -1447,7 +1448,6 @@ bool MYSQL_LOG::appendv(const char* buf, uint len,...)
     new_file(0);
 
 err:
-  pthread_mutex_unlock(&LOCK_log);
   if (!error)
     signal_update();
   DBUG_RETURN(error);

sql/log_event.cc

@@ -2962,13 +2962,39 @@ void Rotate_log_event::print(FILE* file, bool short_form, LAST_EVENT_INFO* last_
 #endif /* MYSQL_CLIENT */
 
 
+
 /*
-  Rotate_log_event::Rotate_log_event()
+  Rotate_log_event::Rotate_log_event() (2 constructors)
 */
 
+
+#ifndef MYSQL_CLIENT
+Rotate_log_event::Rotate_log_event(THD* thd_arg,
+                                   const char* new_log_ident_arg,
+                                   uint ident_len_arg, ulonglong pos_arg,
+                                   uint flags_arg)
+  :Log_event(), new_log_ident(new_log_ident_arg),
+   pos(pos_arg),ident_len(ident_len_arg ? ident_len_arg :
+                          (uint) strlen(new_log_ident_arg)), flags(flags_arg)
+{
+#ifndef DBUG_OFF
+  char buff[22];
+  DBUG_ENTER("Rotate_log_event::Rotate_log_event(THD*,...)");
+  DBUG_PRINT("enter",("new_log_ident %s pos %s flags %lu", new_log_ident_arg,
+                      llstr(pos_arg, buff), flags));
+#endif
+  if (flags & DUP_NAME)
+    new_log_ident= my_strdup_with_length(new_log_ident_arg,
+                                         ident_len,
+                                         MYF(MY_WME));
+  DBUG_VOID_RETURN;
+}
+#endif
+
+
 Rotate_log_event::Rotate_log_event(const char* buf, uint event_len,
                                    const Format_description_log_event* description_event)
-  :Log_event(buf, description_event) ,new_log_ident(NULL),alloced(0)
+  :Log_event(buf, description_event) ,new_log_ident(0), flags(DUP_NAME)
 {
   DBUG_ENTER("Rotate_log_event::Rotate_log_event(char*,...)");
   // The caller will ensure that event_len is what we have at EVENT_LEN_OFFSET
@@ -2983,12 +3009,9 @@ Rotate_log_event::Rotate_log_event(const char* buf, uint event_len,
                      (header_size+post_header_len)); 
   ident_offset = post_header_len; 
   set_if_smaller(ident_len,FN_REFLEN-1);
-  if (!(new_log_ident= my_strdup_with_length((byte*) buf +
-					     ident_offset,
-					     (uint) ident_len,
-					     MYF(MY_WME))))
-    DBUG_VOID_RETURN;
-  alloced = 1;
+  new_log_ident= my_strdup_with_length((byte*) buf + ident_offset,
+                                       (uint) ident_len,
+                                       MYF(MY_WME));
   DBUG_VOID_RETURN;
 }
 

sql/log_event.h

@@ -1247,18 +1247,17 @@ public:
 class Rotate_log_event: public Log_event
 {
 public:
+  enum {
+    DUP_NAME= 2 // if constructor should dup the string argument
+  };
   const char* new_log_ident;
   ulonglong pos;
   uint ident_len;
-  bool alloced;
+  uint flags;
 #ifndef MYSQL_CLIENT
   Rotate_log_event(THD* thd_arg, const char* new_log_ident_arg,
-		   uint ident_len_arg = 0,
-		   ulonglong pos_arg = LOG_EVENT_OFFSET)
-    :Log_event(), new_log_ident(new_log_ident_arg),
-    pos(pos_arg),ident_len(ident_len_arg ? ident_len_arg :
-			   (uint) strlen(new_log_ident_arg)), alloced(0)
-  {}
+		   uint ident_len_arg,
+		   ulonglong pos_arg, uint flags);
 #ifdef HAVE_REPLICATION
   void pack_info(Protocol* protocol);
   int exec_event(struct st_relay_log_info* rli);
@@ -1271,8 +1270,8 @@ public:
                    const Format_description_log_event* description_event);
   ~Rotate_log_event()
   {
-    if (alloced)
-      my_free((gptr) new_log_ident, MYF(0));
+    if (flags & DUP_NAME)
+      my_free((gptr) new_log_ident, MYF(MY_ALLOW_ZERO_PTR));
   }
   Log_event_type get_type_code() { return ROTATE_EVENT;}
   int get_data_size() { return  ident_len + ROTATE_HEADER_LEN;}

sql/slave.cc

@@ -1722,9 +1722,26 @@ static int init_relay_log_info(RELAY_LOG_INFO* rli,
   {
     char buf[FN_REFLEN];
     const char *ln;
+    static bool name_warning_sent= 0;
     ln= rli->relay_log.generate_name(opt_relay_logname, "-relay-bin",
                                      1, buf);
-
+    /* We send the warning only at startup, not after every RESET SLAVE */
+    if (!opt_relay_logname && !opt_relaylog_index_name && !name_warning_sent)
+    {
+      /*
+        User didn't give us info to name the relay log index file.
+        Picking `hostname`-relay-bin.index like we do, causes replication to
+        fail if this slave's hostname is changed later. So, we would like to
+        instead require a name. But as we don't want to break many existing
+        setups, we only give warning, not error.
+      */
+      sql_print_warning("Neither --relay-log nor --relay-log-index were used;"
+                        " so replication "
+                        "may break when this MySQL server acts as a "
+                        "slave and has his hostname changed!! Please "
+                        "use '--relay-log=%s' to avoid this problem.", ln);
+      name_warning_sent= 1;
+    }
     /*
       note, that if open() fails, we'll still have index file open
       but a destructor will take care of that
@@ -1948,6 +1965,55 @@ static int count_relay_log_space(RELAY_LOG_INFO* rli)
 }
 
 
+/*
+  Builds a Rotate from the ignored events' info and writes it to relay log.
+
+  SYNOPSIS
+  write_ignored_events_info_to_relay_log()
+    thd             pointer to I/O thread's thd
+    mi
+
+  DESCRIPTION
+    Slave I/O thread, going to die, must leave a durable trace of the
+    ignored events' end position for the use of the slave SQL thread, by
+    calling this function. Only that thread can call it (see assertion).
+ */
+static void write_ignored_events_info_to_relay_log(THD *thd, MASTER_INFO *mi)
+{
+  RELAY_LOG_INFO *rli= &mi->rli;
+  pthread_mutex_t *log_lock= rli->relay_log.get_log_lock();
+  DBUG_ASSERT(thd == mi->io_thd);
+  pthread_mutex_lock(log_lock);
+  if (rli->ign_master_log_name_end[0])
+  {
+    DBUG_PRINT("info",("writing a Rotate event to track down ignored events"));
+    Rotate_log_event *ev= new Rotate_log_event(thd, rli->ign_master_log_name_end,
+                                               0, rli->ign_master_log_pos_end,
+                                               Rotate_log_event::DUP_NAME);
+    rli->ign_master_log_name_end[0]= 0;
+    /* can unlock before writing as slave SQL thd will soon see our Rotate */
+    pthread_mutex_unlock(log_lock);
+    if (likely((bool)ev))
+    {
+      ev->server_id= 0; // don't be ignored by slave SQL thread
+      if (unlikely(rli->relay_log.append(ev)))
+        sql_print_error("Slave I/O thread failed to write a Rotate event"
+                        " to the relay log, "
+                        "SHOW SLAVE STATUS may be inaccurate");
+      rli->relay_log.harvest_bytes_written(&rli->log_space_total);
+      flush_master_info(mi, 1);
+      delete ev;
+    }
+    else
+      sql_print_error("Slave I/O thread failed to create a Rotate event"
+                      " (out of memory?), "
+                      "SHOW SLAVE STATUS may be inaccurate");
+  }
+  else
+    pthread_mutex_unlock(log_lock);
+}
+
+
 void init_master_info_with_options(MASTER_INFO* mi)
 {
   mi->master_log_name[0] = 0;
@@ -2544,7 +2610,7 @@ st_relay_log_info::st_relay_log_info()
 {
   group_relay_log_name[0]= event_relay_log_name[0]=
     group_master_log_name[0]= 0;
-  last_slave_error[0]=0; until_log_name[0]= 0;
+  last_slave_error[0]= until_log_name[0]= ign_master_log_name_end[0]= 0;
 
   bzero((char*) &info_file, sizeof(info_file));
   bzero((char*) &cache_buf, sizeof(cache_buf));
@@ -3137,12 +3203,20 @@ static int exec_relay_log_event(THD* thd, RELAY_LOG_INFO* rli)
      wait for something for example inside of next_event().
    */
   pthread_mutex_lock(&rli->data_lock);
-
+  /*
+    This tests if the position of the end of the last previous executed event
+    hits the UNTIL barrier.
+    We would prefer to test if the position of the start (or possibly) end of
+    the to-be-read event hits the UNTIL barrier, this is different if there
+    was an event ignored by the I/O thread just before (BUG#13861 to be
+    fixed).
+  */
   if (rli->until_condition!=RELAY_LOG_INFO::UNTIL_NONE &&
       rli->is_until_satisfied())
   {
+    char buf[22];
     sql_print_error("Slave SQL thread stopped because it reached its"
-                    " UNTIL position %ld", (long) rli->until_pos());
+                    " UNTIL position %s", llstr(rli->until_pos(), buf));
     /*
       Setting abort_slave flag because we do not want additional message about
       error in query execution to be printed.
@@ -3330,6 +3404,7 @@ pthread_handler_t handle_slave_io(void *arg)
   THD *thd; // needs to be first for thread_stack
   MYSQL *mysql;
   MASTER_INFO *mi = (MASTER_INFO*)arg;
+  RELAY_LOG_INFO *rli= &mi->rli;
   char llbuff[22];
   uint retry_count;
 
@@ -3572,16 +3647,16 @@ reconnect done to recover from failed read");
         char llbuf1[22], llbuf2[22];
         DBUG_PRINT("info", ("log_space_limit=%s log_space_total=%s \
 ignore_log_space_limit=%d",
-                            llstr(mi->rli.log_space_limit,llbuf1),
-                            llstr(mi->rli.log_space_total,llbuf2),
-                            (int) mi->rli.ignore_log_space_limit)); 
+                            llstr(rli->log_space_limit,llbuf1),
+                            llstr(rli->log_space_total,llbuf2),
+                            (int) rli->ignore_log_space_limit)); 
       }
 #endif
 
-      if (mi->rli.log_space_limit && mi->rli.log_space_limit <
-	  mi->rli.log_space_total &&
-          !mi->rli.ignore_log_space_limit)
-	if (wait_for_relay_log_space(&mi->rli))
+      if (rli->log_space_limit && rli->log_space_limit <
+	  rli->log_space_total &&
+          !rli->ignore_log_space_limit)
+	if (wait_for_relay_log_space(rli))
 	{
 	  sql_print_error("Slave I/O thread aborted while waiting for relay \
 log space");
@@ -3612,6 +3687,7 @@ err:
     mysql_close(mysql);
     mi->mysql=0;
   }
+  write_ignored_events_info_to_relay_log(thd, mi);
   thd->proc_info = "Waiting for slave mutex on exit";
   pthread_mutex_lock(&mi->run_lock);
   mi->slave_running = 0;
@@ -3996,6 +4072,7 @@ static int process_io_rotate(MASTER_INFO *mi, Rotate_log_event *rev)
   if (unlikely(!rev->is_valid()))
     DBUG_RETURN(1);
 
+  /* Safe copy as 'rev' has been "sanitized" in Rotate_log_event's ctor */
   memcpy(mi->master_log_name, rev->new_log_ident, rev->ident_len+1);
   mi->master_log_pos= rev->pos;
   DBUG_PRINT("info", ("master_log_pos: '%s' %d",
@@ -4246,6 +4323,7 @@ int queue_event(MASTER_INFO* mi,const char* buf, ulong event_len)
   int error= 0;
   ulong inc_pos;
   RELAY_LOG_INFO *rli= &mi->rli;
+  pthread_mutex_t *log_lock= rli->relay_log.get_log_lock();
   DBUG_ENTER("queue_event");
 
   if (mi->rli.relay_log.description_event_for_queue->binlog_version<4 &&
@@ -4254,11 +4332,6 @@ int queue_event(MASTER_INFO* mi,const char* buf, ulong event_len)
 
   pthread_mutex_lock(&mi->data_lock);
 
-  /*
-    TODO: figure out if other events in addition to Rotate
-    require special processing.
-    Guilhem 2003-06 : I don't think so.
-  */
   switch (buf[EVENT_TYPE_OFFSET]) {
   case STOP_EVENT:
     /*
@@ -4343,14 +4416,21 @@ int queue_event(MASTER_INFO* mi,const char* buf, ulong event_len)
      direct master (an unsupported, useless setup!).
   */
 
+  pthread_mutex_lock(log_lock);
+
   if ((uint4korr(buf + SERVER_ID_OFFSET) == ::server_id) &&
       !replicate_same_server_id)
   {
     /*
       Do not write it to the relay log.
-      We still want to increment, so that we won't re-read this event from the
-      master if the slave IO thread is now stopped/restarted (more efficient if
-      the events we are ignoring are big LOAD DATA INFILE).
+      a) We still want to increment mi->master_log_pos, so that we won't
+      re-read this event from the master if the slave IO thread is now
+      stopped/restarted (more efficient if the events we are ignoring are big
+      LOAD DATA INFILE).
+      b) We want to record that we are skipping events, for the information of
+      the slave SQL thread, otherwise that thread may let
+      rli->group_relay_log_pos stay too small if the last binlog's event is
+      ignored.
       But events which were generated by this slave and which do not exist in
       the master's binlog (i.e. Format_desc, Rotate & Stop) should not increment
       mi->master_log_pos.
@@ -4358,7 +4438,13 @@ int queue_event(MASTER_INFO* mi,const char* buf, ulong event_len)
     if (buf[EVENT_TYPE_OFFSET]!=FORMAT_DESCRIPTION_EVENT &&
         buf[EVENT_TYPE_OFFSET]!=ROTATE_EVENT &&
         buf[EVENT_TYPE_OFFSET]!=STOP_EVENT)
+    {
       mi->master_log_pos+= inc_pos;
+      memcpy(rli->ign_master_log_name_end, mi->master_log_name, FN_REFLEN);
+      DBUG_ASSERT(rli->ign_master_log_name_end[0]);
+      rli->ign_master_log_pos_end= mi->master_log_pos;
+    }
+    rli->relay_log.signal_update(); // the slave SQL thread needs to re-check
     DBUG_PRINT("info", ("master_log_pos: %d, event originating from the same server, ignored", (ulong) mi->master_log_pos));
   }  
   else
@@ -4371,8 +4457,11 @@ int queue_event(MASTER_INFO* mi,const char* buf, ulong event_len)
       rli->relay_log.harvest_bytes_written(&rli->log_space_total);
     }
     else
-      error=3;
+      error= 3;
+    rli->ign_master_log_name_end[0]= 0; // last event is not ignored
   }
+  pthread_mutex_unlock(log_lock);
+
 
 err:
   pthread_mutex_unlock(&mi->data_lock);
@@ -4756,6 +4845,26 @@ Log_event* next_event(RELAY_LOG_INFO* rli)
         rli->last_master_timestamp= 0;
 
 	DBUG_ASSERT(rli->relay_log.get_open_count() == rli->cur_log_old_open_count);
+
+        if (rli->ign_master_log_name_end[0])
+        {
+          /* We generate and return a Rotate, to make our positions advance */
+          DBUG_PRINT("info",("seeing an ignored end segment"));
+          ev= new Rotate_log_event(thd, rli->ign_master_log_name_end,
+                                   0, rli->ign_master_log_pos_end,
+                                   Rotate_log_event::DUP_NAME);
+          rli->ign_master_log_name_end[0]= 0;
+          if (unlikely(!ev))
+          {
+            errmsg= "Slave SQL thread failed to create a Rotate event "
+              "(out of memory?), SHOW SLAVE STATUS may be inaccurate";
+            goto err;
+          }
+          pthread_mutex_unlock(log_lock);
+          ev->server_id= 0; // don't be ignored by slave SQL thread
+          DBUG_RETURN(ev);
+        }
+
 	/*
 	  We can, and should release data_lock while we are waiting for
 	  update. If we do not, show slave status will block

sql/slave.h

@@ -302,6 +302,17 @@ typedef struct st_relay_log_info
   */
   ulong trans_retries, retried_trans;
 
+  /*
+    If the end of the hot relay log is made of master's events ignored by the
+    slave I/O thread, these two keep track of the coords (in the master's
+    binlog) of the last of these events seen by the slave I/O thread. If not,
+    ign_master_log_name_end[0] == 0.
+    As they are like a Rotate event read/written from/to the relay log, they
+    are both protected by rli->relay_log.LOCK_log.
+  */
+  char ign_master_log_name_end[FN_REFLEN];
+  ulonglong ign_master_log_pos_end;
+
   st_relay_log_info();
   ~st_relay_log_info();