From 62b62319bf92e9c00da8498fe6e9944a4e43ed7d Mon Sep 17 00:00:00 2001
From: Alexander Barkov <bar@mariadb.org>
Date: Tue, 23 May 2017 23:11:31 +0400
Subject: [PATCH] A cleanup for the patch for MDEV-12852, MDEV-12853,
 MDEV-12869

The patch broke expressions like CAST(1.0e+300 AS SIGNED INT)
in binary protocol, e.g.:
  mtr --ps cast

Short real numbers like 1.0e+300 can return huge values,
so using args[0]->max_length is not reliable to choose properly the result
type for Item_func_signed and Item_func_unsigned (between INT and BIGINT).

Setting Item_[un]signed_typecast::max_length to MAX_BIGINT_WIDTH
when doing CAST from FLOAT/DOUBLE, to force type_handler() return
&type_handler_longlong rather than &type_handler_long.
---
 sql/item_func.h |  4 ++++
 sql/sql_type.cc | 15 +++++++++++++++
 sql/sql_type.h  |  2 ++
 3 files changed, 21 insertions(+)

diff --git a/sql/item_func.h b/sql/item_func.h
index 9055a9e5e98..a6a00a9b162 100644
--- a/sql/item_func.h
+++ b/sql/item_func.h
@@ -848,6 +848,10 @@ public:
     null_value= args[0]->null_value;
     return value;
   }
+  void fix_length_and_dec_double()
+  {
+    fix_char_length(MAX_BIGINT_WIDTH);
+  }
   void fix_length_and_dec_generic()
   {
     uint32 char_length= MY_MIN(args[0]->max_char_length(),
diff --git a/sql/sql_type.cc b/sql/sql_type.cc
index 504fab21845..05563a2d171 100644
--- a/sql/sql_type.cc
+++ b/sql/sql_type.cc
@@ -4214,6 +4214,21 @@ bool Type_handler_string_result::
   return false;
 }
 
+bool Type_handler_real_result::
+       Item_func_signed_fix_length_and_dec(Item_func_signed *item) const
+{
+  item->fix_length_and_dec_double();
+  return false;
+}
+
+
+bool Type_handler_real_result::
+       Item_func_unsigned_fix_length_and_dec(Item_func_unsigned *item) const
+{
+  item->fix_length_and_dec_double();
+  return false;
+}
+
 
 bool Type_handler::
        Item_double_typecast_fix_length_and_dec(Item_double_typecast *item) const
diff --git a/sql/sql_type.h b/sql/sql_type.h
index aeaec7ff95f..c377f267dd0 100644
--- a/sql/sql_type.h
+++ b/sql/sql_type.h
@@ -1299,6 +1299,8 @@ public:
   bool Item_sum_sum_fix_length_and_dec(Item_sum_sum *) const;
   bool Item_sum_avg_fix_length_and_dec(Item_sum_avg *) const;
   bool Item_sum_variance_fix_length_and_dec(Item_sum_variance *) const;
+  bool Item_func_signed_fix_length_and_dec(Item_func_signed *item) const;
+  bool Item_func_unsigned_fix_length_and_dec(Item_func_unsigned *item) const;
   longlong Item_val_int_signed_typecast(Item *item) const;
   longlong Item_val_int_unsigned_typecast(Item *item) const;
   String *Item_func_hex_val_str_ascii(Item_func_hex *item, String *str) const;