InnoDB: Some low-level optimizations made based on OProfile results.

innobase/dict/dict0mem.c:
  dict_mem_table_create(): Add a debug assertion.
innobase/include/btr0btr.ic:
  btr_node_ptr_get_child_page_no(): Add a UNIV_UNLIKELY hint.
  Remove a buf_frame_align() call.
innobase/include/btr0cur.ic:
  btr_cur_get_page(): Add a debug assertion.
innobase/include/buf0buf.ic:
  buf_block_peek_if_too_old(): Replace if() with return().
  buf_block_align(), buf_frame_align(): Add UNIV_UNLIKELY hints.
innobase/include/data0type.ic:
  dtype_get_fixed_size(): Add UNIV_UNLIKELY hints.
innobase/include/mem0mem.ic:
  Remove signedness warning in debug assertion.
innobase/include/read0read.ic:
  read_view_sees_trx_id(): Eliminate a comparison inside loop.
innobase/include/row0sel.ic:
  open_step(): Add UNIV_EXPECT hint.
innobase/include/row0upd.ic:
  upd_field_set_field_no(): Add a UNIV_UNLIKELY hint.
innobase/include/sync0rw.ic:
  Add UNIV_LIKELY and UNIV_UNLIKELY hints.
  rw_lock_x_lock_func_nowait(): Eliminate a function call.
  Replace ut_a() assertions with ut_ad().
innobase/include/trx0rseg.ic:
  Add UNIV_UNLIKELY hints.
innobase/include/ut0rnd.ic:
  ut_fold_binary(): Eliminate a loop variable
  to avoid register spilling on x86.

innobase/dict/dict0mem.c

@@ -42,6 +42,7 @@ dict_mem_table_create(
 	mem_heap_t*	heap;
 	
 	ut_ad(name);
+	ut_ad(comp == FALSE || comp == TRUE);
 
 	heap = mem_heap_create(DICT_HEAP_SIZE);
 

innobase/include/btr0btr.ic

@@ -200,10 +200,10 @@ btr_node_ptr_get_child_page_no(
 	
 	page_no = mach_read_from_4(field);
 
-	if (page_no == 0) {
+	if (UNIV_UNLIKELY(page_no == 0)) {
 		fprintf(stderr,
 "InnoDB: a nonsensical page number 0 in a node ptr record at offset %lu\n",
-		       (unsigned long)(rec - buf_frame_align(rec)));
+			(ulong) ut_align_offset(rec, UNIV_PAGE_SIZE));
 		buf_page_print(buf_frame_align(rec));
 	}
 

innobase/include/btr0cur.ic

@@ -52,7 +52,9 @@ btr_cur_get_page(
 				/* out: pointer to page */
 	btr_cur_t*	cursor)	/* in: tree cursor */
 {
-	return(buf_frame_align(page_cur_get_rec(&(cursor->page_cur))));
+	page_t*	page = buf_frame_align(page_cur_get_rec(&(cursor->page_cur)));
+	ut_ad(!!page_is_comp(page) == cursor->index->table->comp);
+	return(page);
 }
 
 /*************************************************************

innobase/include/buf0buf.ic

@@ -26,12 +26,8 @@ buf_block_peek_if_too_old(
 				/* out: TRUE if should be made younger */
 	buf_block_t*	block)	/* in: block to make younger */
 {
-	if (buf_pool->freed_page_clock >= block->freed_page_clock 
-				+ 1 + (buf_pool->curr_size / 1024)) {
-		return(TRUE);
-	}
-
-	return(FALSE);
+	return(buf_pool->freed_page_clock >= block->freed_page_clock
+				+ 1 + (buf_pool->curr_size / 1024));
 }
 
 /*************************************************************************
@@ -210,8 +206,8 @@ buf_block_align(
 
 	frame_zero = buf_pool->frame_zero;
 
-	if ((ulint)ptr < (ulint)frame_zero
-	    || (ulint)ptr > (ulint)(buf_pool->high_end)) {
+	if (UNIV_UNLIKELY((ulint)ptr < (ulint)frame_zero)
+	    || UNIV_UNLIKELY((ulint)ptr > (ulint)(buf_pool->high_end))) {
 
 		ut_print_timestamp(stderr);	
 		fprintf(stderr,
@@ -246,8 +242,8 @@ buf_frame_align(
 
 	frame = ut_align_down(ptr, UNIV_PAGE_SIZE);
 
-	if (((ulint)frame < (ulint)(buf_pool->frame_zero))
-	    || (ulint)frame >= (ulint)(buf_pool->high_end)) {
+	if (UNIV_UNLIKELY((ulint)frame < (ulint)(buf_pool->frame_zero))
+	    || UNIV_UNLIKELY((ulint)frame >= (ulint)(buf_pool->high_end))) {
 
 		ut_print_timestamp(stderr);	
 		fprintf(stderr,

innobase/include/data0type.ic

@@ -388,8 +388,8 @@ dtype_get_fixed_size(
 					dtype_get_charset_coll(type->prtype),
 					&mbminlen, &mbmaxlen);
 
-				if (type->mbminlen != mbminlen
-					|| type->mbmaxlen != mbmaxlen) {
+				if (UNIV_UNLIKELY(type->mbminlen != mbminlen)
+				|| UNIV_UNLIKELY(type->mbmaxlen != mbmaxlen)) {
 
 					ut_print_timestamp(stderr);
 					fprintf(stderr, "  InnoDB: "

innobase/include/mem0mem.ic

@@ -623,7 +623,7 @@ mem_strdupq(
 	}
 	*d++ = q;
 	*d++ = '\0';
-	ut_ad(len == d - dst);
+	ut_ad((ssize_t) len == d - dst);
 	return(dst);
 }
 

innobase/include/read0read.ic

@@ -71,13 +71,8 @@ read_view_sees_trx_id(
 
 		cmp = ut_dulint_cmp(trx_id,
 				read_view_get_nth_trx_id(view, n_ids - i - 1));
-		if (0 == cmp) {
-
-			return(FALSE);
-
-		} else if (cmp < 0) {
-
-			return(TRUE);
+		if (cmp <= 0) {
+			return(cmp < 0);
 		}
 	}
 	

innobase/include/row0sel.ic

@@ -75,7 +75,7 @@ open_step(
 		}
 	}
 			
-	if (err != DB_SUCCESS) {
+	if (UNIV_EXPECT(err, DB_SUCCESS) != DB_SUCCESS) {
 		/* SQL error detected */
 		fprintf(stderr, "SQL error %lu\n", (ulong) err);
 

innobase/include/row0upd.ic

@@ -83,7 +83,7 @@ upd_field_set_field_no(
 {	
 	upd_field->field_no = field_no;
 
-	if (field_no >= dict_index_get_n_fields(index)) {
+	if (UNIV_UNLIKELY(field_no >= dict_index_get_n_fields(index))) {
 		fprintf(stderr,
 			"InnoDB: Error: trying to access field %lu in ",
 			(ulong) field_no);

innobase/include/sync0rw.ic

@@ -138,7 +138,7 @@ rw_lock_s_lock_low(
 #endif /* UNIV_SYNC_DEBUG */
 	/* Check if the writer field is free */
 
-	if (lock->writer == RW_LOCK_NOT_LOCKED) {
+	if (UNIV_LIKELY(lock->writer == RW_LOCK_NOT_LOCKED)) {
 		/* Set the shared lock by incrementing the reader count */
 		lock->reader_count++;
 
@@ -243,7 +243,7 @@ rw_lock_s_lock_func(
 
 	mutex_enter(rw_lock_get_mutex(lock));
 
-	if (TRUE == rw_lock_s_lock_low(lock, pass, file_name, line)) {
+	if (UNIV_LIKELY(rw_lock_s_lock_low(lock, pass, file_name, line))) {
 		mutex_exit(rw_lock_get_mutex(lock));
 
 		return; /* Success */
@@ -307,21 +307,18 @@ rw_lock_x_lock_func_nowait(
 	const char*	file_name,/* in: file name where lock requested */
 	ulint		line)	/* in: line where requested */
 {
-	ibool	success	= FALSE;
-	
+	ibool		success		= FALSE;
+	os_thread_id_t	curr_thread	= os_thread_get_curr_id();
 	mutex_enter(rw_lock_get_mutex(lock));
 
-	if ((rw_lock_get_reader_count(lock) == 0)
-	     && ((rw_lock_get_writer(lock) == RW_LOCK_NOT_LOCKED)
-	     	 || ((rw_lock_get_writer(lock) == RW_LOCK_EX)
-	     	     && (lock->pass == 0)
-	     	     && os_thread_eq(lock->writer_thread,
-					os_thread_get_curr_id())))) {
-
+	if (UNIV_UNLIKELY(rw_lock_get_reader_count(lock) != 0)) {
+	} else if (UNIV_LIKELY(rw_lock_get_writer(lock)
+			       == RW_LOCK_NOT_LOCKED)) {
 		rw_lock_set_writer(lock, RW_LOCK_EX);
-		lock->writer_thread = os_thread_get_curr_id();
-		lock->writer_count++;
+		lock->writer_thread = curr_thread;
 		lock->pass = 0;
+	relock:
+		lock->writer_count++;
 			
 #ifdef UNIV_SYNC_DEBUG
 		rw_lock_add_debug_info(lock, 0, RW_LOCK_EX, file_name, line);
@@ -331,6 +328,10 @@ rw_lock_x_lock_func_nowait(
 		lock->last_x_line = line;
 
 		success = TRUE;
+	} else if (rw_lock_get_writer(lock) == RW_LOCK_EX
+			&& lock->pass == 0
+			&& os_thread_eq(lock->writer_thread, curr_thread)) {
+		goto relock;
 	}
 
 	mutex_exit(rw_lock_get_mutex(lock));
@@ -361,7 +362,7 @@ rw_lock_s_unlock_func(
 
 	/* Reset the shared lock by decrementing the reader count */
 
-	ut_a(lock->reader_count > 0);
+	ut_ad(lock->reader_count > 0);
 	lock->reader_count--;
 
 #ifdef UNIV_SYNC_DEBUG
@@ -371,7 +372,8 @@ rw_lock_s_unlock_func(
 	/* If there may be waiters and this was the last s-lock,
 	signal the object */
 
-	if (lock->waiters && (lock->reader_count == 0)) {
+	if (UNIV_UNLIKELY(lock->waiters)
+			&& lock->reader_count == 0) {
 	       	sg = TRUE;
 
 		rw_lock_set_waiters(lock, 0);
@@ -379,7 +381,7 @@ rw_lock_s_unlock_func(
 	
 	mutex_exit(mutex);
 
-	if (sg == TRUE) {
+	if (UNIV_UNLIKELY(sg)) {
 		sync_array_signal_object(sync_primary_wait_array, lock);
 	}
 
@@ -450,7 +452,8 @@ rw_lock_x_unlock_func(
 #endif
 	
 	/* If there may be waiters, signal the lock */
-	if (lock->waiters && (lock->writer_count == 0)) {
+	if (UNIV_UNLIKELY(lock->waiters)
+			&& lock->writer_count == 0) {
 
 	       	sg = TRUE;
 		rw_lock_set_waiters(lock, 0);
@@ -458,7 +461,7 @@ rw_lock_x_unlock_func(
 	
 	mutex_exit(&(lock->mutex));
 
-	if (sg == TRUE) {
+	if (UNIV_UNLIKELY(sg)) {
 		sync_array_signal_object(sync_primary_wait_array, lock);
 	}
 

innobase/include/trx0rseg.ic

@@ -65,7 +65,7 @@ trx_rsegf_get_nth_undo(
 	ulint		n,	/* in: index of slot */
 	mtr_t*		mtr)	/* in: mtr */
 {
-	if (n >= TRX_RSEG_N_SLOTS) {
+	if (UNIV_UNLIKELY(n >= TRX_RSEG_N_SLOTS)) {
 		fprintf(stderr,
 		"InnoDB: Error: trying to get slot %lu of rseg\n", (unsigned long) n);
 		ut_error;
@@ -86,7 +86,7 @@ trx_rsegf_set_nth_undo(
 	ulint		page_no,/* in: page number of the undo log segment */
 	mtr_t*		mtr)	/* in: mtr */
 {
-	if (n >= TRX_RSEG_N_SLOTS) {
+	if (UNIV_UNLIKELY(n >= TRX_RSEG_N_SLOTS)) {
 		fprintf(stderr,
 		"InnoDB: Error: trying to set slot %lu of rseg\n", (unsigned long) n);
 		ut_error;

innobase/include/ut0rnd.ic

@@ -207,12 +207,12 @@ ut_fold_binary(
 	const byte*	str,	/* in: string of bytes */
 	ulint		len)	/* in: length */
 {
-	ulint	i;
-	ulint	fold = 0;
+	const byte*	str_end	= str + len;
+	ulint		fold = 0;
 
 	ut_ad(str);
 
-	for (i = 0; i < len; i++) {
+	while (str < str_end) {
 		fold = ut_fold_ulint_pair(fold, (ulint)(*str));
 
 		str++;