mariadb/sql/my_json_writer.cc
Marko Mäkelä 94d0bb4dbe MDEV-20377: Make WITH_MSAN more usable
MemorySanitizer (clang -fsanitize=memory) requires that all code
be compiled with instrumentation enabled. The C runtime library
is an exception. Failure to use instrumented libraries will cause
bogus messages about memory being uninitialized.

In WITH_MSAN builds, we must avoid calling getservbyname(),
because even though it is a standard library function, it is
not instrumented, not even in clang 10.

The following cmake options were tested:

-DCMAKE_C_FLAGS='-march=native -O2'
-DCMAKE_CXX_FLAGS='-stdlib=libc++ -march=native -O2'
-DWITH_EMBEDDED_SERVER=OFF -DWITH_UNIT_TESTS=OFF -DCMAKE_BUILD_TYPE=Debug
-DWITH_INNODB_{BZIP2,LZ4,LZMA,LZO,SNAPPY}=OFF
-DPLUGIN_{ARCHIVE,TOKUDB,MROONGA,OQGRAPH,ROCKSDB,CONNECT,SPIDER}=NO
-DWITH_SAFEMALLOC=OFF
-DWITH_{ZLIB,SSL,PCRE}=bundled
-DHAVE_LIBAIO_H=0
-DWITH_MSAN=ON

MEM_MAKE_DEFINED(): An alias for VALGRIND_MAKE_MEM_DEFINED()
and in the future, __msan_unpoison().

For now, neither MEM_MAKE_DEFINED() nor MEM_UNDEFINED()
perform any action under MSAN. Enabling them will catch more bugs, but
will also require some more fixes or work-arounds.

Json_writer::add_double(): Work around a frequently occurring
failure in optimizer tests, related to EXPLAIN FORMAT=JSON.

dtoa(): Disable MSAN altogether. For some reason, this function
is triggering a lot of trouble, especially when invoked for
DBUG functions. The MDL default timeout is dd=86400 seconds,
and for some reason it is claimed to be uninitialized.

InnoDB: Define UNIV_DEBUG_VALGRIND also WITH_MSAN.

ut_crc32_8_hw(), ut_crc32_64_low_hw(): Use the compiler built-in
functions instead of inline assembler when building WITH_MSAN.
This will require at least -msse4.2 when building for IA-32 or AMD64.
The inline assembler would not be instrumented, and would thus cause
bogus failures.
2020-03-28 21:36:30 +02:00

439 lines
8.6 KiB
C++

/* Copyright (C) 2014, 2020, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */
#include "mariadb.h"
#include "sql_priv.h"
#include "sql_string.h"
#include "my_json_writer.h"
void Json_writer::append_indent()
{
if (!document_start)
output.append('\n');
for (int i=0; i< indent_level; i++)
output.append(' ');
}
void Json_writer::start_object()
{
fmt_helper.on_start_object();
if (!element_started)
start_element();
output.append("{");
indent_level+=INDENT_SIZE;
first_child=true;
element_started= false;
document_start= false;
}
void Json_writer::start_array()
{
if (fmt_helper.on_start_array())
return;
if (!element_started)
start_element();
output.append("[");
indent_level+=INDENT_SIZE;
first_child=true;
element_started= false;
document_start= false;
}
void Json_writer::end_object()
{
indent_level-=INDENT_SIZE;
if (!first_child)
append_indent();
first_child= false;
output.append("}");
}
void Json_writer::end_array()
{
if (fmt_helper.on_end_array())
return;
indent_level-=INDENT_SIZE;
if (!first_child)
append_indent();
output.append("]");
}
Json_writer& Json_writer::add_member(const char *name)
{
size_t len= strlen(name);
if (fmt_helper.on_add_member(name, len))
return *this; // handled
// assert that we are in an object
DBUG_ASSERT(!element_started);
start_element();
output.append('"');
output.append(name, len);
output.append("\": ", 3);
return *this;
}
Json_writer& Json_writer::add_member(const char *name, size_t len)
{
if (fmt_helper.on_add_member(name, len))
return *this; // handled
// assert that we are in an object
DBUG_ASSERT(!element_started);
start_element();
output.append('"');
output.append(name, len);
output.append("\": ");
return *this;
}
/*
Used by formatting helper to print something that is formatted by the helper.
We should only separate it from the previous element.
*/
void Json_writer::start_sub_element()
{
//element_started= true;
if (first_child)
first_child= false;
else
output.append(',');
append_indent();
}
void Json_writer::start_element()
{
element_started= true;
if (first_child)
first_child= false;
else
output.append(',');
append_indent();
}
void Json_writer::add_ll(longlong val)
{
char buf[64];
my_snprintf(buf, sizeof(buf), "%lld", val);
add_unquoted_str(buf);
}
void Json_writer::add_ull(ulonglong val)
{
char buf[64];
my_snprintf(buf, sizeof(buf), "%llu", val);
add_unquoted_str(buf);
}
/* Add a memory size, printing in Kb, Kb, Gb if necessary */
void Json_writer::add_size(longlong val)
{
char buf[64];
size_t len;
if (val < 1024)
len= my_snprintf(buf, sizeof(buf), "%lld", val);
else if (val < 1024*1024*16)
{
/* Values less than 16MB are specified in KB for precision */
len= my_snprintf(buf, sizeof(buf), "%lld", val/1024);
strcpy(buf + len, "Kb");
len+= 2;
}
else
{
len= my_snprintf(buf, sizeof(buf), "%lld", val/(1024*1024));
strcpy(buf + len, "Mb");
len+= 2;
}
add_str(buf, len);
}
void Json_writer::add_double(double val)
{
char buf[64];
#if __has_feature(memory_sanitizer) // FIXME: remove this workaround for
__msan_unpoison(&val, sizeof val); // main.range_mrr_icp & many other tests
#endif
size_t len= my_snprintf(buf, sizeof(buf), "%lg", val);
add_unquoted_str(buf, len);
}
void Json_writer::add_bool(bool val)
{
add_unquoted_str(val? "true" : "false");
}
void Json_writer::add_null()
{
add_unquoted_str("null", (size_t) 4);
}
void Json_writer::add_unquoted_str(const char* str)
{
size_t len= strlen(str);
if (fmt_helper.on_add_str(str, len))
return;
if (!element_started)
start_element();
output.append(str, len);
element_started= false;
}
void Json_writer::add_unquoted_str(const char* str, size_t len)
{
if (fmt_helper.on_add_str(str, len))
return;
if (!element_started)
start_element();
output.append(str, len);
element_started= false;
}
void Json_writer::add_str(const char *str)
{
size_t len= strlen(str);
if (fmt_helper.on_add_str(str, len))
return;
if (!element_started)
start_element();
output.append('"');
output.append(str, len);
output.append('"');
element_started= false;
}
/*
This function is used to add only num_bytes of str to the output string
*/
void Json_writer::add_str(const char* str, size_t num_bytes)
{
if (fmt_helper.on_add_str(str, num_bytes))
return;
if (!element_started)
start_element();
output.append('"');
output.append(str, num_bytes);
output.append('"');
element_started= false;
}
void Json_writer::add_str(const String &str)
{
add_str(str.ptr(), str.length());
}
Json_writer_temp_disable::Json_writer_temp_disable(THD *thd_arg)
{
thd= thd_arg;
thd->opt_trace.disable_tracing_if_required();
}
Json_writer_temp_disable::~Json_writer_temp_disable()
{
thd->opt_trace.enable_tracing_if_required();
}
bool Single_line_formatting_helper::on_add_member(const char *name,
size_t len)
{
DBUG_ASSERT(state== INACTIVE || state == DISABLED);
if (state != DISABLED)
{
// remove everything from the array
buf_ptr= buffer;
//append member name to the array
if (len < MAX_LINE_LEN)
{
memcpy(buf_ptr, name, len);
buf_ptr+=len;
*(buf_ptr++)= 0;
line_len= owner->indent_level + (uint)len + 1;
state= ADD_MEMBER;
return true; // handled
}
}
return false; // not handled
}
bool Single_line_formatting_helper::on_start_array()
{
if (state == ADD_MEMBER)
{
state= IN_ARRAY;
return true; // handled
}
else
{
if (state != DISABLED)
state= INACTIVE;
// TODO: what if we have accumulated some stuff already? shouldn't we
// flush it?
return false; // not handled
}
}
bool Single_line_formatting_helper::on_end_array()
{
if (state == IN_ARRAY)
{
flush_on_one_line();
state= INACTIVE;
return true; // handled
}
return false; // not handled
}
void Single_line_formatting_helper::on_start_object()
{
// Nested objects will not be printed on one line
disable_and_flush();
}
bool Single_line_formatting_helper::on_add_str(const char *str,
size_t len)
{
if (state == IN_ARRAY)
{
// New length will be:
// "$string",
// quote + quote + comma + space = 4
if (line_len + len + 4 > MAX_LINE_LEN)
{
disable_and_flush();
return false; // didn't handle the last element
}
//append string to array
memcpy(buf_ptr, str, len);
buf_ptr+=len;
*(buf_ptr++)= 0;
line_len += (uint)len + 4;
return true; // handled
}
disable_and_flush();
return false; // not handled
}
/*
Append everything accumulated to the output on one line
*/
void Single_line_formatting_helper::flush_on_one_line()
{
owner->start_sub_element();
char *ptr= buffer;
int nr= 0;
while (ptr < buf_ptr)
{
char *str= ptr;
if (nr == 0)
{
owner->output.append('"');
owner->output.append(str);
owner->output.append("\": ");
owner->output.append('[');
}
else
{
if (nr != 1)
owner->output.append(", ");
owner->output.append('"');
owner->output.append(str);
owner->output.append('"');
}
nr++;
while (*ptr!=0)
ptr++;
ptr++;
}
owner->output.append(']');
/* We've printed out the contents of the buffer, mark it as empty */
buf_ptr= buffer;
}
void Single_line_formatting_helper::disable_and_flush()
{
if (state == DISABLED)
return;
bool start_array= (state == IN_ARRAY);
state= DISABLED;
// deactivate ourselves and flush all accumulated calls.
char *ptr= buffer;
int nr= 0;
while (ptr < buf_ptr)
{
char *str= ptr;
size_t len= strlen(str);
if (nr == 0)
{
owner->add_member(str, len);
if (start_array)
owner->start_array();
}
else
{
//if (nr == 1)
// owner->start_array();
owner->add_str(str, len);
}
nr++;
ptr+= len+1;
}
buf_ptr= buffer;
state= INACTIVE;
}