Bug#9112 - Merge table with composite index producing invalid results with some queries

The problem was an ab-use of last_rkey_length.
Formerly we saved the packed key length (of the search key)
in this element. But in certain cases it got replaced by 
the (packed) result key length.
Now we use a new element of MI_INFO to save the packed key 
length of the search key.


myisam/mi_dbug.c:
  Bug#9112 - Merge table with composite index producing invalid results with some queries
  Fixed the recognition of NULL values in _mi_print_key().
myisam/mi_rkey.c:
  Bug#9112 - Merge table with composite index producing invalid results with some queries
  Saved the packed key length in a new element of MI_INFO.
myisam/mi_search.c:
  Bug#9112 - Merge table with composite index producing invalid results with some queries
  Added a comment and trace prints.
myisam/myisamdef.h:
  Bug#9112 - Merge table with composite index producing invalid results with some queries
  Added a new element to store the packed key length
  for use by the MyISAMMRG engine.
myisammrg/myrg_rkey.c:
  Bug#9112 - Merge table with composite index producing invalid results with some queries
  Changed to use the new element of MI_INFO to get at the
  packed key length.
mysql-test/r/merge.result:
  Bug#9112 - Merge table with composite index producing invalid results with some queries
  The test result.
mysql-test/t/merge.test:
  Bug#9112 - Merge table with composite index producing invalid results with some queries
  The test case.

myisam/mi_dbug.c

@@ -40,12 +40,12 @@ void _mi_print_key(FILE *stream, register MI_KEYSEG *keyseg,
     end= key+ keyseg->length;
     if (keyseg->flag & HA_NULL_PART)
     {
-      if (!*key)
+      /* A NULL value is encoded by a 1-byte flag. Zero means NULL. */
+      if (! *(key++))
       {
 	fprintf(stream,"NULL");
 	continue;
       }
-      key++;
     }
 
     switch (keyseg->type) {

myisam/mi_rkey.c

@@ -31,8 +31,8 @@ int mi_rkey(MI_INFO *info, byte *buf, int inx, const byte *key, uint key_len,
   MI_KEYSEG *last_used_keyseg;
   uint pack_key_length, use_key_length, nextflag;
   DBUG_ENTER("mi_rkey");
-  DBUG_PRINT("enter",("base: %lx  inx: %d  search_flag: %d",
-		      info,inx,search_flag));
+  DBUG_PRINT("enter", ("base: %p  buf: %p  inx: %d  search_flag: %d",
+                       info, buf, inx, search_flag));
 
   if ((inx = _mi_check_index(info,inx)) < 0)
     DBUG_RETURN(my_errno);
@@ -44,9 +44,12 @@ int mi_rkey(MI_INFO *info, byte *buf, int inx, const byte *key, uint key_len,
   {
     if (key_len == 0)
       key_len=USE_WHOLE_KEY;
+    /* Save the packed key for later use in the second buffer of lastkey. */
     key_buff=info->lastkey+info->s->base.max_key_length;
     pack_key_length=_mi_pack_key(info, (uint) inx, key_buff, (uchar*) key,
 				 key_len, &last_used_keyseg);
+    /* Save packed_key_length for use by the MERGE engine. */
+    info->pack_key_length= pack_key_length;
     DBUG_EXECUTE("key",_mi_print_key(DBUG_FILE, keyinfo->seg,
 				     key_buff, pack_key_length););
   }

myisam/mi_search.c

@@ -1237,11 +1237,21 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag,
   reg1 MI_KEYSEG *keyseg;
   uchar *start_key,*page,*page_end,*from,*from_end;
   uint length,tmp;
+  DBUG_ENTER("_mi_get_binary_pack_key");
 
   page= *page_pos;
   page_end=page+MI_MAX_KEY_BUFF+1;
   start_key=key;
 
+  /*
+    Keys are compressed the following way:
+
+    prefix length    Packed length of prefix for the prev key. (1 or 3 bytes)
+    for each key segment:
+      [is null]        Null indicator if can be null (1 byte, zero means null)
+      [length]         Packed length if varlength (1 or 3 bytes)
+    pointer          Reference to the data file (last_keyseg->length).
+  */
   get_key_length(length,page);
   if (length)
   {
@@ -1251,7 +1261,7 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag,
                           length, keyinfo->maxlength, *page_pos));
       DBUG_DUMP("key",(char*) *page_pos,16);
       my_errno=HA_ERR_CRASHED;
-      return 0;                                 /* Wrong key */
+      DBUG_RETURN(0);                                 /* Wrong key */
     }
     from=key;  from_end=key+length;
   }
@@ -1312,12 +1322,12 @@ uint _mi_get_binary_pack_key(register MI_KEYDEF *keyinfo, uint nod_flag,
     {
       DBUG_PRINT("error",("Error when unpacking key"));
       my_errno=HA_ERR_CRASHED;
-      return 0;                                 /* Error */
+      DBUG_RETURN(0);                                 /* Error */
     }
     memcpy((byte*) key,(byte*) from,(size_t) length);
     *page_pos= from+length;
   }
-  return((uint) (key-start_key)+keyseg->length);
+  DBUG_RETURN((uint) (key-start_key)+keyseg->length);
 }
 
 

myisam/myisamdef.h

@@ -255,6 +255,7 @@ struct st_myisam_info {
   uint	lastkey_length;			/* Length of key in lastkey */
   uint	last_rkey_length;		/* Last length in mi_rkey() */
   uint  save_lastkey_length;
+  uint  pack_key_length;                /* For MYISAMMRG */
   int	errkey;				/* Got last error on this key */
   int   lock_type;			/* How database was locked */
   int   tmp_lock_type;			/* When locked by readinfo */