From a736e63f7c494b0611a27a8951bf8fe0980cbaad Mon Sep 17 00:00:00 2001 From: Olivier Bertrand Date: Sun, 22 Feb 2015 17:53:02 +0100 Subject: [PATCH] - Add new Json UDF's Json_Array_Add, Json_Array_Grp and Json_Object_Grp. Handle longjmp's raised during json processing. modified: storage/connect/global.h storage/connect/ha_connect.cc storage/connect/json.cpp storage/connect/jsonudf.cpp - Fix wrong references to the suppressed g->Trace variables. modified: storage/connect/global.h storage/connect/plugutil.c storage/connect/tabjson.cpp storage/connect/tabodbc.cpp --- storage/connect/global.h | 2 +- storage/connect/ha_connect.cc | 16 ++++- storage/connect/json.cpp | 6 ++ storage/connect/jsonudf.cpp | 112 ++++++++++++++++++++++++++++++---- storage/connect/plgdbutl.cpp | 2 +- storage/connect/plugutil.c | 2 +- storage/connect/tabjson.cpp | 6 +- storage/connect/tabodbc.cpp | 8 +-- 8 files changed, 131 insertions(+), 23 deletions(-) diff --git a/storage/connect/global.h b/storage/connect/global.h index 88e5094d6d2..a67bb605755 100644 --- a/storage/connect/global.h +++ b/storage/connect/global.h @@ -235,7 +235,7 @@ typedef struct _global { /* Global structure */ void *Xchk; /* indexes in create/alter */ short Alchecked; /* Checked for ALTER */ short Mrr; /* True when doing mrr */ - short Trace; + int N; /* Utility */ int jump_level; jmp_buf jumper[MAX_JUMP + 2]; } GLOBAL; diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 125b6921e12..e73a64a150c 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -165,9 +165,10 @@ /***********************************************************************/ /* Initialize the ha_connect static members. */ /***********************************************************************/ -#define SZCONV 8192 -#define SZWORK 67108864 // Default work area size 64M -#define SZWMIN 4194304 // Minimum work area size 4M +#define SZCONV 8192 +#define SZWORK 67108864 // Default work area size 64M +#define SZWMIN 4194304 // Minimum work area size 4M +#define JSONMAX 10 // JSON Default max grp size extern "C" { char version[]= "Version 1.03.0006 February 06, 2015"; @@ -217,6 +218,7 @@ bool ExactInfo(void); USETEMP UseTemp(void); int GetConvSize(void); TYPCONV GetTypeConv(void); +uint GetJsonGrpSize(void); uint GetWorkSize(void); void SetWorkSize(uint); extern "C" const char *msglang(void); @@ -323,6 +325,12 @@ static MYSQL_THDVAR_ENUM( 0, // def (no) &xconv_typelib); // typelib +// Estimate max number of rows for JSON aggregate functions +static MYSQL_THDVAR_UINT(json_grp_size, + PLUGIN_VAR_RQCMDARG, // opt + "max number of rows for JSON aggregate functions.", + NULL, NULL, JSONMAX, 1, INT_MAX, 1); + #if defined(XMSG) || defined(NEWMSG) const char *language_names[]= { @@ -353,6 +361,7 @@ bool ExactInfo(void) {return THDVAR(current_thd, exact_info);} USETEMP UseTemp(void) {return (USETEMP)THDVAR(current_thd, use_tempfile);} int GetConvSize(void) {return THDVAR(current_thd, conv_size);} TYPCONV GetTypeConv(void) {return (TYPCONV)THDVAR(current_thd, type_conv);} +uint GetJsonGrpSize(void) {return THDVAR(current_thd, json_grp_size);} uint GetWorkSize(void) {return THDVAR(current_thd, work_size);} void SetWorkSize(uint n) { @@ -6516,6 +6525,7 @@ static struct st_mysql_sys_var* connect_system_variables[]= { #if defined(XMSG) MYSQL_SYSVAR(errmsg_dir_path), #endif // XMSG + MYSQL_SYSVAR(json_grp_size), NULL }; diff --git a/storage/connect/json.cpp b/storage/connect/json.cpp index 251ba762a53..8031ba51b19 100644 --- a/storage/connect/json.cpp +++ b/storage/connect/json.cpp @@ -74,6 +74,7 @@ PJSON ParseJson(PGLOBAL g, char *s, int len, int pretty, bool *comma) goto err; } else if (!(jsp = ParseObject(g, ++i, src))) goto err; + break; case ' ': case '\t': @@ -90,6 +91,11 @@ PJSON ParseJson(PGLOBAL g, char *s, int len, int pretty, bool *comma) sprintf(g->Message, "Unexpected ',' (pretty=%d)", pretty); goto err; + case '"': + if (!(jsp = ParseValue(g, i, src))) + goto err; + + break; case '(': b = true; break; diff --git a/storage/connect/jsonudf.cpp b/storage/connect/jsonudf.cpp index 6ad1cbeea79..b100f377295 100644 --- a/storage/connect/jsonudf.cpp +++ b/storage/connect/jsonudf.cpp @@ -8,12 +8,18 @@ /* Include relevant sections of the MariaDB header file. */ /***********************************************************************/ #include +#include #include +#include #include "global.h" #include "plgdbsem.h" #include "json.h" +#define MEMFIX 512 + +uint GetJsonGrpSize(void); + extern "C" { DllExport my_bool Json_Value_init(UDF_INIT*, UDF_ARGS*, char*); DllExport char *Json_Value(UDF_INIT*, UDF_ARGS*, char*, @@ -23,6 +29,10 @@ DllExport my_bool Json_Array_init(UDF_INIT*, UDF_ARGS*, char*); DllExport char *Json_Array(UDF_INIT*, UDF_ARGS*, char*, unsigned long*, char *, char *); DllExport void Json_Array_deinit(UDF_INIT*); +DllExport my_bool Json_Array_Add_init(UDF_INIT*, UDF_ARGS*, char*); +DllExport char *Json_Array_Add(UDF_INIT*, UDF_ARGS*, char*, + unsigned long*, char *, char *); +DllExport void Json_Array_Add_deinit(UDF_INIT*); DllExport my_bool Json_Object_init(UDF_INIT*, UDF_ARGS*, char*); DllExport char *Json_Object(UDF_INIT*, UDF_ARGS*, char*, unsigned long*, char *, char *); @@ -44,8 +54,8 @@ DllExport void Json_Object_Grp_deinit(UDF_INIT*); /***********************************************************************/ /* Allocate and initialise the memory area. */ /***********************************************************************/ -static my_bool JsonInit(UDF_INIT *initid, char *message, unsigned long reslen, - unsigned long memlen) +static my_bool JsonInit(UDF_INIT *initid, char *message, + unsigned long reslen, unsigned long memlen) { PGLOBAL g = PlugInit(NULL, memlen); @@ -119,7 +129,7 @@ static my_bool CalcLen(UDF_ARGS *args, my_bool obj, } // endfor i // Calculate the amount of memory needed - memlen = 1024 + sizeof(JOUTSTR) + reslen; + memlen = MEMFIX + sizeof(JOUTSTR) + reslen; for (i = 0; i < args->arg_count; i++) { memlen += (args->lengths[i] + sizeof(JVALUE)); @@ -219,14 +229,23 @@ static PSZ MakeKey(PGLOBAL g, UDF_ARGS *args, int i) static PJVAL MakeValue(PGLOBAL g, UDF_ARGS *args, int i) { char *sap = args->args[i]; + PJSON jsp; PJVAL jvp = new(g) JVALUE; if (sap) switch (args->arg_type[i]) { case STRING_RESULT: if (args->lengths[i]) { - if (IsJson(args, i)) - jvp->SetValue(ParseJson(g, sap, args->lengths[i], 0)); - else + if (IsJson(args, i)) { + if (!(jsp = ParseJson(g, sap, args->lengths[i], 0))) + push_warning(current_thd, Sql_condition::WARN_LEVEL_WARN, 0, + g->Message); + + if (jsp && jsp->GetType() == TYPE_JVAL) + jvp = (PJVAL)jsp; + else + jvp->SetValue(jsp); + + } else jvp->SetString(g, MakePSZ(g, args, i)); } // endif str @@ -328,6 +347,59 @@ void Json_Array_deinit(UDF_INIT* initid) PlugExit((PGLOBAL)initid->ptr); } // end of Json_Array_deinit +/***********************************************************************/ +/* Add values to a Json array. */ +/***********************************************************************/ +my_bool Json_Array_Add_init(UDF_INIT *initid, UDF_ARGS *args, char *message) +{ + unsigned long reslen, memlen; + + if (args->arg_count < 2) { + strcpy(message, "Json_Value_Add must have at least 2 arguments"); + return true; + } else if (!IsJson(args, 0)) { + strcpy(message, "Json_Value_Add first argument must be a json array"); + return true; + } else + CalcLen(args, false, reslen, memlen); + + return JsonInit(initid, message, reslen, memlen); +} // end of Json_Array_Add_init + +char *Json_Array_Add(UDF_INIT *initid, UDF_ARGS *args, char *result, + unsigned long *res_length, char *is_null, char *error) +{ + char *str; + PJVAL jvp; + PJAR arp; + PGLOBAL g = (PGLOBAL)initid->ptr; + + PlugSubSet(g, g->Sarea, g->Sarea_Size); + jvp = MakeValue(g, args, 0); + + if (jvp->GetValType() != TYPE_JAR) { + arp = new(g) JARRAY; + arp->AddValue(g, jvp); + } else + arp = jvp->GetArray(); + + for (uint i = 1; i < args->arg_count; i++) + arp->AddValue(g, MakeValue(g, args, i)); + + arp->InitArray(g); + + if (!(str = Serialize(g, arp, NULL, 0))) + str = strcpy(result, g->Message); + + *res_length = strlen(str); + return str; +} // end of Json_Array_Add + +void Json_Array_Add_deinit(UDF_INIT* initid) +{ + PlugExit((PGLOBAL)initid->ptr); +} // end of Json_Array_Add_deinit + /***********************************************************************/ /* Make a Json Oject containing all the parameters. */ /***********************************************************************/ @@ -370,7 +442,7 @@ void Json_Object_deinit(UDF_INIT* initid) /***********************************************************************/ my_bool Json_Array_Grp_init(UDF_INIT *initid, UDF_ARGS *args, char *message) { - unsigned long reslen, memlen, n = 10; + unsigned long reslen, memlen, n = GetJsonGrpSize(); if (args->arg_count != 1) { strcpy(message, "Json_Array_Grp can only accept 1 argument"); @@ -379,7 +451,7 @@ my_bool Json_Array_Grp_init(UDF_INIT *initid, UDF_ARGS *args, char *message) CalcLen(args, false, reslen, memlen); reslen *= n; - memlen *= n; + memlen += ((memlen - MEMFIX) * (n - 1)); if (JsonInit(initid, message, reslen, memlen)) return true; @@ -388,6 +460,7 @@ my_bool Json_Array_Grp_init(UDF_INIT *initid, UDF_ARGS *args, char *message) PlugSubSet(g, g->Sarea, g->Sarea_Size); g->Activityp = (PACTIVITY)new(g) JARRAY; + g->N = (int)n; return false; } // end of Json_Array_Grp_init @@ -397,7 +470,9 @@ void Json_Array_Grp_add(UDF_INIT *initid, UDF_ARGS *args, PGLOBAL g = (PGLOBAL)initid->ptr; PJAR arp = (PJAR)g->Activityp; - arp->AddValue(g, MakeValue(g, args, 0)); + if (g->N-- > 0) + arp->AddValue(g, MakeValue(g, args, 0)); + } // end of Json_Array_Grp_add char *Json_Array_Grp(UDF_INIT *initid, UDF_ARGS *args, char *result, @@ -407,6 +482,10 @@ char *Json_Array_Grp(UDF_INIT *initid, UDF_ARGS *args, char *result, PGLOBAL g = (PGLOBAL)initid->ptr; PJAR arp = (PJAR)g->Activityp; + if (g->N < 0) + push_warning(current_thd, Sql_condition::WARN_LEVEL_WARN, 0, + "Result truncated to json_grp_size values"); + arp->InitArray(g); if (!(str = Serialize(g, arp, NULL, 0))) @@ -422,6 +501,7 @@ void Json_Array_Grp_clear(UDF_INIT *initid, char *is_null, char *error) PlugSubSet(g, g->Sarea, g->Sarea_Size); g->Activityp = (PACTIVITY)new(g) JARRAY; + g->N = GetJsonGrpSize(); } // end of Json_Array_Grp_clear void Json_Array_Grp_deinit(UDF_INIT* initid) @@ -434,7 +514,7 @@ void Json_Array_Grp_deinit(UDF_INIT* initid) /***********************************************************************/ my_bool Json_Object_Grp_init(UDF_INIT *initid, UDF_ARGS *args, char *message) { - unsigned long reslen, memlen, n = 10; + unsigned long reslen, memlen, n = GetJsonGrpSize(); if (args->arg_count != 2) { strcpy(message, "Json_Array_Grp can only accept 2 argument"); @@ -443,7 +523,7 @@ my_bool Json_Object_Grp_init(UDF_INIT *initid, UDF_ARGS *args, char *message) CalcLen(args, true, reslen, memlen); reslen *= n; - memlen *= n; + memlen += ((memlen - MEMFIX) * (n - 1)); if (JsonInit(initid, message, reslen, memlen)) return true; @@ -452,6 +532,7 @@ my_bool Json_Object_Grp_init(UDF_INIT *initid, UDF_ARGS *args, char *message) PlugSubSet(g, g->Sarea, g->Sarea_Size); g->Activityp = (PACTIVITY)new(g) JOBJECT; + g->N = (int)n; return false; } // end of Json_Object_Grp_init @@ -461,7 +542,9 @@ void Json_Object_Grp_add(UDF_INIT *initid, UDF_ARGS *args, PGLOBAL g = (PGLOBAL)initid->ptr; PJOB objp = (PJOB)g->Activityp; - objp->SetValue(g, MakeValue(g, args, 0), MakePSZ(g, args, 1)); + if (g->N-- > 0) + objp->SetValue(g, MakeValue(g, args, 0), MakePSZ(g, args, 1)); + } // end of Json_Object_Grp_add char *Json_Object_Grp(UDF_INIT *initid, UDF_ARGS *args, char *result, @@ -471,6 +554,10 @@ char *Json_Object_Grp(UDF_INIT *initid, UDF_ARGS *args, char *result, PGLOBAL g = (PGLOBAL)initid->ptr; PJOB objp = (PJOB)g->Activityp; + if (g->N < 0) + push_warning(current_thd, Sql_condition::WARN_LEVEL_WARN, 0, + "Result truncated to json_grp_size values"); + if (!(str = Serialize(g, objp, NULL, 0))) str = strcpy(result, g->Message); @@ -484,6 +571,7 @@ void Json_Object_Grp_clear(UDF_INIT *initid, char *is_null, char *error) PlugSubSet(g, g->Sarea, g->Sarea_Size); g->Activityp = (PACTIVITY)new(g) JOBJECT; + g->N = GetJsonGrpSize(); } // end of Json_Object_Grp_clear void Json_Object_Grp_deinit(UDF_INIT* initid) diff --git a/storage/connect/plgdbutl.cpp b/storage/connect/plgdbutl.cpp index d8009bcc71f..d5429e8a344 100644 --- a/storage/connect/plgdbutl.cpp +++ b/storage/connect/plgdbutl.cpp @@ -311,7 +311,7 @@ PQRYRES PlgAllocResult(PGLOBAL g, int ncol, int maxres, int ids, else crp->Kdata = NULL; - if (g->Trace) + if (trace) htrc("Column(%d) %s type=%d len=%d value=%p\n", crp->Ncol, crp->Name, crp->Type, crp->Length, crp->Kdata); diff --git a/storage/connect/plugutil.c b/storage/connect/plugutil.c index c77975e5e30..18b48b8eccc 100644 --- a/storage/connect/plugutil.c +++ b/storage/connect/plugutil.c @@ -144,12 +144,12 @@ PGLOBAL PlugInit(LPCSTR Language, uint worksize) return NULL; } else { g->Sarea_Size = worksize; - g->Trace = 0; g->Createas = 0; g->Alchecked = 0; g->Mrr = 0; g->Activityp = g->ActivityStart = NULL; g->Xchk = NULL; + g->N = 0; strcpy(g->Message, ""); /*******************************************************************/ diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 362b28ba79b..f99b0d2ef1d 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -931,7 +931,11 @@ void JSONCOL::WriteColumn(PGLOBAL g) case TYPE_STRING: if (Nodes[Nod-1].Op == OP_XX) { s = Value->GetCharValue(); - jsp = ParseJson(g, s, (int)strlen(s), 0); + + if (!(jsp = ParseJson(g, s, (int)strlen(s), 0))) { + strcpy(g->Message, s); + longjmp(g->jumper[g->jump_level], 666); + } // endif jsp if (arp) { if (Nod > 1 && Nodes[Nod-2].Rank) diff --git a/storage/connect/tabodbc.cpp b/storage/connect/tabodbc.cpp index cb18870bc55..bf76a124b70 100644 --- a/storage/connect/tabodbc.cpp +++ b/storage/connect/tabodbc.cpp @@ -787,7 +787,7 @@ bool TDBODBC::OpenDB(PGLOBAL g) { bool rc = true; - if (g->Trace) + if (trace) htrc("ODBC OpenDB: tdbp=%p tdb=R%d use=%dmode=%d\n", this, Tdb_No, Use, Mode); @@ -1185,12 +1185,12 @@ void ODBCCOL::ReadColumn(PGLOBAL g) } // endif Buf_Type - if (g->Trace) { + if (trace) { char buf[64]; htrc("ODBC Column %s: rows=%d buf=%p type=%d value=%s\n", Name, tdbp->Rows, Bufp, Buf_Type, Value->GetCharString(buf)); - } // endif Trace + } // endif trace put: if (tdbp->Memory != 2) @@ -1424,7 +1424,7 @@ bool TDBXDBC::OpenDB(PGLOBAL g) { bool rc = false; - if (g->Trace) + if (trace) htrc("ODBC OpenDB: tdbp=%p tdb=R%d use=%dmode=%d\n", this, Tdb_No, Use, Mode);