From 0ce02f6d98b5eb94797a935e06fa89f37a241b52 Mon Sep 17 00:00:00 2001
From: unknown <ramil/ram@mysql.com/myoffice.izhnet.ru>
Date: Wed, 6 Dec 2006 16:32:12 +0400
Subject: [PATCH] Fix for bug #22533: Traditional: Too-long bit value not
 rejected.

Problem: storing >=8 byte hexadecimal values we don't check data.
Fix: check if the data fits the {u}longlong range.


mysql-test/r/select.result:
  Fix for bug #22533: Traditional: Too-long bit value not rejected.
    - test result.
mysql-test/t/range.test:
  Fix for bug #22533: Traditional: Too-long bit value not rejected.
    - adjusted.
mysql-test/t/select.test:
  Fix for bug #22533: Traditional: Too-long bit value not rejected.
    - test case.
sql/item.cc:
  Fix for bug #22533: Traditional: Too-long bit value not rejected.
    - limit storing value to {U}LONGLONG_MAX in numeric context.
---
 mysql-test/r/select.result | 17 +++++++++++++++++
 mysql-test/t/range.test    |  4 ++--
 mysql-test/t/select.test   | 13 ++++++++++++-
 sql/item.cc                | 25 +++++++++++++++++++------
 4 files changed, 50 insertions(+), 9 deletions(-)

diff --git a/mysql-test/r/select.result b/mysql-test/r/select.result
index f09143fcaa6..6dc971a953c 100644
--- a/mysql-test/r/select.result
+++ b/mysql-test/r/select.result
@@ -2819,3 +2819,20 @@ select min(key1) from t1 where key1 >= 0.3762 and rand() + 0.5 >= 0.5;
 min(key1)
 0.37619999051094
 DROP TABLE t1,t2;
+create table t1(a bigint unsigned, b bigint);
+insert into t1 values (0xfffffffffffffffff, 0xfffffffffffffffff), 
+(0x10000000000000000, 0x10000000000000000), 
+(0x8fffffffffffffff, 0x8fffffffffffffff);
+Warnings:
+Warning	1264	Data truncated; out of range for column 'a' at row 1
+Warning	1264	Data truncated; out of range for column 'b' at row 1
+Warning	1264	Data truncated; out of range for column 'a' at row 2
+Warning	1264	Data truncated; out of range for column 'b' at row 2
+Warning	1264	Data truncated; out of range for column 'b' at row 3
+select hex(a), hex(b) from t1;
+hex(a)	hex(b)
+FFFFFFFFFFFFFFFF	7FFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFF	7FFFFFFFFFFFFFFF
+8FFFFFFFFFFFFFFF	7FFFFFFFFFFFFFFF
+drop table t1;
+End of 4.1 tests
diff --git a/mysql-test/t/range.test b/mysql-test/t/range.test
index 245178d7d4a..6d3b2fb52ee 100644
--- a/mysql-test/t/range.test
+++ b/mysql-test/t/range.test
@@ -400,8 +400,8 @@ select count(*) from t1 where x = 18446744073709551601;
 
 
 create table t2 (x bigint not null);
-insert into t2(x) values (0xfffffffffffffff0);
-insert into t2(x) values (0xfffffffffffffff1);
+insert into t2(x) values (-16);
+insert into t2(x) values (-15);
 select * from t2;
 select count(*) from t2 where x>0;
 select count(*) from t2 where x=0;
diff --git a/mysql-test/t/select.test b/mysql-test/t/select.test
index 3f9fb59d26f..0dc179e9b4b 100644
--- a/mysql-test/t/select.test
+++ b/mysql-test/t/select.test
@@ -2342,4 +2342,15 @@ select min(key1) from t1 where key1 >= 0.3762 and rand() + 0.5 >= 0.5;
 DROP TABLE t1,t2;
 --enable_ps_protocol
 
-# End of 4.1 tests
+#
+# Bug #22533: storing large hex strings
+#
+
+create table t1(a bigint unsigned, b bigint);
+insert into t1 values (0xfffffffffffffffff, 0xfffffffffffffffff), 
+  (0x10000000000000000, 0x10000000000000000), 
+  (0x8fffffffffffffff, 0x8fffffffffffffff);
+select hex(a), hex(b) from t1;
+drop table t1;
+
+--echo End of 4.1 tests
diff --git a/sql/item.cc b/sql/item.cc
index 94f0a24fcc3..30f34f9ec3b 100644
--- a/sql/item.cc
+++ b/sql/item.cc
@@ -2340,18 +2340,31 @@ longlong Item_varbinary::val_int()
 
 int Item_varbinary::save_in_field(Field *field, bool no_conversions)
 {
-  int error;
   field->set_notnull();
   if (field->result_type() == STRING_RESULT)
+    return field->store(str_value.ptr(), str_value.length(), 
+                        collation.collation);
+
+  ulonglong nr;
+  uint32 length= str_value.length();
+  if (length > 8)
   {
-    error=field->store(str_value.ptr(),str_value.length(),collation.collation);
+    nr= field->flags & UNSIGNED_FLAG ? ULONGLONG_MAX : LONGLONG_MAX;
+    goto warn;
   }
-  else
+  nr= (ulonglong) val_int();
+  if ((length == 8) && !(field->flags & UNSIGNED_FLAG) && (nr > LONGLONG_MAX))
   {
-    longlong nr=val_int();
-    error=field->store(nr);
+    nr= LONGLONG_MAX;
+    goto warn;
   }
-  return error;
+  return field->store((longlong) nr);
+
+warn:
+  if (!field->store((longlong) nr))
+    field->set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_OUT_OF_RANGE,
+                       1);
+  return 1;
 }