From 1e33cfb36a84b477a468dbcfc1ccc3035a9efb81 Mon Sep 17 00:00:00 2001
From: unknown <holyfoot/hf@mysql.com/hfmain.(none)>
Date: Wed, 16 May 2007 10:12:49 +0500
Subject: [PATCH] bug #8663 cant use bigint unsigned as input to cast in the
 case of the overflow in the decimal->integer conversion we didn't return the
 proper boundary value, but just the result of the conversion we calculated on
 the moment of the error

mysql-test/r/bigint.result:
  bug #8663 cant use bigint unsigned as input to cast
  test result fixed
mysql-test/t/bigint.test:
  bug #8663 cant use bigint unsigned as input to cast
  test case
strings/decimal.c:
  bug #8663 cant use bigint unsigned as input to cast
  decimal->int conversion fixed to return proper boundary value
  in the case of overflow
---
 mysql-test/r/bigint.result | 10 ++++++++++
 mysql-test/t/bigint.test   |  6 ++++++
 strings/decimal.c          |  6 +++++-
 3 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/mysql-test/r/bigint.result b/mysql-test/r/bigint.result
index 541a15561e2..f18d1c9b583 100644
--- a/mysql-test/r/bigint.result
+++ b/mysql-test/r/bigint.result
@@ -352,3 +352,13 @@ select c1 mod 50 as result from t1;
 result
 6
 drop table t1;
+select cast(19999999999999999999 as signed);
+cast(19999999999999999999 as signed)
+9223372036854775807
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+select cast(-19999999999999999999 as signed);
+cast(-19999999999999999999 as signed)
+-9223372036854775808
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
diff --git a/mysql-test/t/bigint.test b/mysql-test/t/bigint.test
index 6c1229db83f..9a5fb11229d 100644
--- a/mysql-test/t/bigint.test
+++ b/mysql-test/t/bigint.test
@@ -288,3 +288,9 @@ insert into t1 values (10000002383263201056);
 select c1 mod 50 as result from t1;
 drop table t1;
 
+#
+# Bug #8663 cant use bgint unsigned as input to cast
+#
+
+select cast(19999999999999999999 as signed);
+select cast(-19999999999999999999 as signed);
diff --git a/strings/decimal.c b/strings/decimal.c
index 65db68b1b59..1ae75167794 100644
--- a/strings/decimal.c
+++ b/strings/decimal.c
@@ -1083,7 +1083,11 @@ int decimal2longlong(decimal_t *from, longlong *to)
     x=x*DIG_BASE - *buf++;
     if (unlikely(y < (LONGLONG_MIN/DIG_BASE) || x > y))
     {
-      *to= from->sign ? y : -y;
+      /*
+        the decimal is bigger than any possible integer
+        return border integer depending on the sign
+      */
+      *to= from->sign ? LONGLONG_MIN : LONGLONG_MAX;
       return E_DEC_OVERFLOW;
     }
   }