From 86c1225699b7ab403398ebfa4863aef80e6d566a Mon Sep 17 00:00:00 2001
From: Sergey Glukhov <sergey.glukhov@oracle.com>
Date: Mon, 28 Mar 2011 12:35:50 +0400
Subject: [PATCH] Bug#11764994  57900: CREATE TABLE .. SELECT ASSERTS SCALE >=
 0 && PRECISION > 0 && SCALE <= PR Assert fails due to overflow which happens
 in Item_func_int_val::fix_num_length_and_dec() as geometry functions have
 max_length value equal to max_field_size(4294967295U). The fix is to skip
 max_length calculation for some boundary cases.

---
 mysql-test/r/func_math.result | 7 +++++++
 mysql-test/t/func_math.test   | 9 +++++++++
 sql/item_func.cc              | 7 ++++---
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/mysql-test/r/func_math.result b/mysql-test/r/func_math.result
index 3a626084c9e..ad0b872145b 100644
--- a/mysql-test/r/func_math.result
+++ b/mysql-test/r/func_math.result
@@ -511,4 +511,11 @@ t1	CREATE TABLE `t1` (
   `C` varchar(23) DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1
 DROP TABLE t1;
+#
+# Bug#11764994  57900: CREATE TABLE .. SELECT ASSERTS SCALE >= 0 && PRECISION > 0 && SCALE <= PR
+#
+CREATE TABLE t1 SELECT CEIL(LINESTRINGFROMWKB(1) DIV NULL);
+DROP TABLE t1;
+CREATE TABLE t1 SELECT FLOOR(LINESTRINGFROMWKB(1) DIV NULL);
+DROP TABLE t1;
 End of 5.1 tests
diff --git a/mysql-test/t/func_math.test b/mysql-test/t/func_math.test
index c8ea11c7490..64b6a3a4ea6 100644
--- a/mysql-test/t/func_math.test
+++ b/mysql-test/t/func_math.test
@@ -324,4 +324,13 @@ CREATE TABLE t1 SELECT CAST((CASE(('')) WHEN (CONVERT(1, CHAR(1))) THEN (('' / 1
 SHOW CREATE TABLE t1;
 DROP TABLE t1;
 
+--echo #
+--echo # Bug#11764994  57900: CREATE TABLE .. SELECT ASSERTS SCALE >= 0 && PRECISION > 0 && SCALE <= PR
+--echo #
+
+CREATE TABLE t1 SELECT CEIL(LINESTRINGFROMWKB(1) DIV NULL);
+DROP TABLE t1;
+CREATE TABLE t1 SELECT FLOOR(LINESTRINGFROMWKB(1) DIV NULL);
+DROP TABLE t1;
+
 --echo End of 5.1 tests
diff --git a/sql/item_func.cc b/sql/item_func.cc
index d4fd2c94e1d..79fa37bd372 100644
--- a/sql/item_func.cc
+++ b/sql/item_func.cc
@@ -1803,9 +1803,10 @@ void Item_func_integer::fix_length_and_dec()
 
 void Item_func_int_val::fix_num_length_and_dec()
 {
-  max_length= args[0]->max_length - (args[0]->decimals ?
-                                     args[0]->decimals + 1 :
-                                     0) + 2;
+  ulonglong tmp_max_length= (ulonglong ) args[0]->max_length - 
+    (args[0]->decimals ? args[0]->decimals + 1 : 0) + 2;
+  max_length= tmp_max_length > (ulonglong) max_field_size ?
+    max_field_size : (uint32) tmp_max_length;
   uint tmp= float_length(decimals);
   set_if_smaller(max_length,tmp);
   decimals= 0;