MDEV-20377: Make WITH_MSAN more usable

MemorySanitizer (clang -fsanitize=memory) requires that all code
be compiled with instrumentation enabled. The C runtime library
is an exception. Failure to use instrumented libraries will cause
bogus messages about memory being uninitialized.

In WITH_MSAN builds, we must avoid calling getservbyname(),
because even though it is a standard library function, it is
not instrumented, not even in clang 10.

The following cmake options were tested:

-DCMAKE_C_FLAGS='-march=native -O2'
-DCMAKE_CXX_FLAGS='-stdlib=libc++ -march=native -O2'
-DWITH_EMBEDDED_SERVER=OFF -DWITH_UNIT_TESTS=OFF -DCMAKE_BUILD_TYPE=Debug
-DWITH_INNODB_{BZIP2,LZ4,LZMA,LZO,SNAPPY}=OFF
-DPLUGIN_{ARCHIVE,TOKUDB,MROONGA,OQGRAPH,ROCKSDB,CONNECT,SPIDER}=NO
-DWITH_SAFEMALLOC=OFF
-DWITH_{ZLIB,SSL,PCRE}=bundled
-DHAVE_LIBAIO_H=0
-DWITH_MSAN=ON

MEM_MAKE_DEFINED(): An alias for VALGRIND_MAKE_MEM_DEFINED()
and in the future, __msan_unpoison().

For now, neither MEM_MAKE_DEFINED() nor MEM_UNDEFINED()
perform any action under MSAN. Enabling them will catch more bugs, but
will also require some more fixes or work-arounds.

Json_writer::add_double(): Work around a frequently occurring
failure in optimizer tests, related to EXPLAIN FORMAT=JSON.

dtoa(): Disable MSAN altogether. For some reason, this function
is triggering a lot of trouble, especially when invoked for
DBUG functions. The MDL default timeout is dd=86400 seconds,
and for some reason it is claimed to be uninitialized.

InnoDB: Define UNIV_DEBUG_VALGRIND also WITH_MSAN.

ut_crc32_8_hw(), ut_crc32_64_low_hw(): Use the compiler built-in
functions instead of inline assembler when building WITH_MSAN.
This will require at least -msse4.2 when building for IA-32 or AMD64.
The inline assembler would not be instrumented, and would thus cause
bogus failures.

include/my_valgrind.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2010, 2019, MariaDB Corporation.
+/* Copyright (C) 2010, 2020, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -33,6 +33,7 @@
 #if defined(HAVE_VALGRIND_MEMCHECK_H) && defined(HAVE_valgrind)
 # include <valgrind/memcheck.h>
 # define MEM_UNDEFINED(a,len) VALGRIND_MAKE_MEM_UNDEFINED(a,len)
+# define MEM_MAKE_DEFINED(a,len) VALGRIND_MAKE_MEM_DEFINED(a,len)
 # define MEM_NOACCESS(a,len) VALGRIND_MAKE_MEM_NOACCESS(a,len)
 # define MEM_CHECK_ADDRESSABLE(a,len) VALGRIND_CHECK_MEM_IS_ADDRESSABLE(a,len)
 # define MEM_CHECK_DEFINED(a,len) VALGRIND_CHECK_MEM_IS_DEFINED(a,len)
@@ -42,12 +43,27 @@
 /* How to do manual poisoning:
 https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */
 # define MEM_UNDEFINED(a,len) ASAN_UNPOISON_MEMORY_REGION(a,len)
+# define MEM_MAKE_DEFINED(a,len) ((void) 0)
 # define MEM_NOACCESS(a,len) ASAN_POISON_MEMORY_REGION(a,len)
 # define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0)
 # define MEM_CHECK_DEFINED(a,len) ((void) 0)
 # define REDZONE_SIZE 8
+#elif __has_feature(memory_sanitizer)
+# include <sanitizer/msan_interface.h>
+# if 0 /* FIXME: these reveal lots of failures */
+#  define MEM_UNDEFINED(a,len) __msan_allocated_memory(a,len)
+#  define MEM_MAKE_DEFINED(a,len) __msan_unpoison(a,len)
+# else
+#  define MEM_UNDEFINED(a,len) ((void) 0)
+#  define MEM_MAKE_DEFINED(a,len) ((void) 0)
+# endif
+# define MEM_NOACCESS(a,len) ((void) 0)
+# define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0)
+# define MEM_CHECK_DEFINED(a,len) __msan_check_mem_is_initialized(a,len)
+# define REDZONE_SIZE 8
 #else
 # define MEM_UNDEFINED(a,len) ((void) (a), (void) (len))
+# define MEM_MAKE_DEFINED(a,len) ((void) 0)
 # define MEM_NOACCESS(a,len) ((void) 0)
 # define MEM_CHECK_ADDRESSABLE(a,len) ((void) 0)
 # define MEM_CHECK_DEFINED(a,len) ((void) 0)

libmariadb

@@ -1 +1 @@
-Subproject commit 1768cb6c322d403c1e372b368cc3c23b660b7930
+Subproject commit 7a2c052ffc8b93acaa1e4de502c83cbae548148c

libmysqld/libmysql.c

@@ -155,8 +155,10 @@ int STDCALL mysql_server_init(int argc __attribute__((unused)),
       */
 
 #if MYSQL_PORT_DEFAULT == 0
+# if !__has_feature(memory_sanitizer) // Work around MSAN deficiency
       if ((serv_ptr= getservbyname("mysql", "tcp")))
         mysql_port= (uint) ntohs((ushort) serv_ptr->s_port);
+# endif
 #endif
       if ((env= getenv("MYSQL_TCP_PORT")))
         mysql_port=(uint) atoi(env);

sql/my_json_writer.cc

@@ -1,4 +1,4 @@
-/* Copyright (C) 2014 SkySQL Ab, MariaDB Corporation Ab
+/* Copyright (C) 2014, 2020, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -180,6 +180,9 @@ void Json_writer::add_size(longlong val)
 void Json_writer::add_double(double val)
 {
   char buf[64];
+#if __has_feature(memory_sanitizer)  // FIXME: remove this workaround for
+  __msan_unpoison(&val, sizeof val); // main.range_mrr_icp & many other tests
+#endif
   size_t len= my_snprintf(buf, sizeof(buf), "%lg", val);
   add_unquoted_str(buf, len);
 }

sql/mysqld.cc

@@ -2174,9 +2174,11 @@ static void set_ports()
     */
 
 #if MYSQL_PORT_DEFAULT == 0
+# if !__has_feature(memory_sanitizer) // Work around MSAN deficiency
     struct  servent *serv_ptr;
     if ((serv_ptr= getservbyname("mysql", "tcp")))
       SYSVAR_AUTOSIZE(mysqld_port, ntohs((u_short) serv_ptr->s_port));
+# endif
 #endif
     if ((env = getenv("MYSQL_TCP_PORT")))
     {

storage/innobase/include/dict0stats.ic

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2020, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -202,18 +202,18 @@ dict_stats_deinit(
 	for (index = dict_table_get_first_index(table);
 	     index != NULL;
 	     index = dict_table_get_next_index(index)) {
-
-		ulint	n_uniq = dict_index_get_n_unique(index);
-
 		UNIV_MEM_INVALID(
 			index->stat_n_diff_key_vals,
-			n_uniq * sizeof(index->stat_n_diff_key_vals[0]));
+			index->n_uniq
+			* sizeof(index->stat_n_diff_key_vals[0]));
 		UNIV_MEM_INVALID(
 			index->stat_n_sample_sizes,
-			n_uniq * sizeof(index->stat_n_sample_sizes[0]));
+			index->n_uniq
+			* sizeof(index->stat_n_sample_sizes[0]));
 		UNIV_MEM_INVALID(
 			index->stat_n_non_null_key_vals,
-			n_uniq * sizeof(index->stat_n_non_null_key_vals[0]));
+			index->n_uniq
+			* sizeof(index->stat_n_non_null_key_vals[0]));
 		UNIV_MEM_INVALID(
 			&index->stat_index_size,
 			sizeof(index->stat_index_size));

storage/innobase/include/univ.i

@@ -170,8 +170,12 @@ using the call command. */
 #define UNIV_ENABLE_UNIT_TEST_ROW_RAW_FORMAT_INT
 */
 
+#include <my_valgrind.h>
+
 #if defined HAVE_valgrind && defined HAVE_VALGRIND_MEMCHECK_H
 # define UNIV_DEBUG_VALGRIND
+#elif __has_feature(memory_sanitizer)
+# define UNIV_DEBUG_VALGRIND
 #endif
 
 #ifdef DBUG_OFF
@@ -573,14 +577,13 @@ typedef void* os_thread_ret_t;
 #include "ut0ut.h"
 #include "sync0types.h"
 
-#include <my_valgrind.h>
 /* define UNIV macros in terms of my_valgrind.h */
 #define UNIV_MEM_INVALID(addr, size) 	MEM_UNDEFINED(addr, size)
 #define UNIV_MEM_FREE(addr, size) 	MEM_NOACCESS(addr, size)
 #define UNIV_MEM_ALLOC(addr, size) 	UNIV_MEM_INVALID(addr, size)
 #ifdef UNIV_DEBUG_VALGRIND
 # include <valgrind/memcheck.h>
-# define UNIV_MEM_VALID(addr, size) VALGRIND_MAKE_MEM_DEFINED(addr, size)
+# define UNIV_MEM_VALID(addr, size) MEM_MAKE_DEFINED(addr, size)
 # define UNIV_MEM_DESC(addr, size) VALGRIND_CREATE_BLOCK(addr, size, #addr)
 # define UNIV_MEM_UNDESC(b) VALGRIND_DISCARD(b)
 # define UNIV_MEM_ASSERT_RW_LOW(addr, size, should_abort) do {		\

storage/innobase/ut/ut0crc32.cc

@@ -172,6 +172,8 @@ ut_crc32_8_hw(
 {
 #  ifdef _MSC_VER
 	*crc = _mm_crc32_u8(*crc, (*data)[0]);
+#  elif __has_feature(memory_sanitizer)
+	*crc = __builtin_ia32_crc32qi(*crc, (*data)[0]);
 #  else
 	asm("crc32b %1, %0"
 	    /* output operands */
@@ -204,6 +206,8 @@ ut_crc32_64_low_hw(
 #   else
 #    error Not Supported processors type.
 #   endif
+#  elif __has_feature(memory_sanitizer)
+	crc_64bit = __builtin_ia32_crc32di(crc_64bit, data);
 #  else
 	asm("crc32q %1, %0"
 	    /* output operands */

strings/dtoa.c

@@ -1,5 +1,5 @@
 /* Copyright (c) 2007, 2012, Oracle and/or its affiliates. All rights reserved.
-   Copyright (c) 2017, MariaDB Corporation.
+   Copyright (c) 2017, 2020, MariaDB Corporation.
 
    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Library General Public
@@ -2168,6 +2168,9 @@ static int quorem(Bigint *b, Bigint *S)
 
 static char *dtoa(double dd, int mode, int ndigits, int *decpt, int *sign,
                   char **rve, char *buf, size_t buf_size)
+#if __has_feature(memory_sanitizer)
+  __attribute__((no_sanitize("memory"))) // FIXME: dd is claimed uninitialized
+#endif
 {
   /*
     Arguments ndigits, decpt, sign are similar to those