BUG#10589: des_encrypt functionality always return NULL

- Push warnings if des_encrypt or des_descrypt function fails because of out of resources or wrong params.
 - Push warning if des_encrypt or des_decrypt function is used when server is missing support for openssl.
 - Add test func_encrypt_nossl that is tun when the server is missing support for openssl.

mysql-test/include/not_openssl.inc

@@ -0,0 +1,4 @@
+-- require r/not_openssl.require
+disable_query_log;
+show variables like "have_openssl";
+enable_query_log;

mysql-test/r/func_encrypt.result

@@ -120,6 +120,60 @@ hello
 select des_decrypt(des_encrypt("hello",4),'password4');
 des_decrypt(des_encrypt("hello",4),'password4')
 hello
+select des_encrypt("hello",10);
+des_encrypt("hello",10)
+NULL
+Warnings:
+Error	1108	Incorrect parameters to procedure 'des_encrypt'
+select des_encrypt(NULL);
+des_encrypt(NULL)
+NULL
+Warnings:
+Error	1108	Incorrect parameters to procedure 'des_encrypt'
+select des_encrypt(NULL, 10);
+des_encrypt(NULL, 10)
+NULL
+Warnings:
+Error	1108	Incorrect parameters to procedure 'des_encrypt'
+select des_encrypt(NULL, NULL);
+des_encrypt(NULL, NULL)
+NULL
+Warnings:
+Error	1108	Incorrect parameters to procedure 'des_encrypt'
+select des_encrypt(10, NULL);
+des_encrypt(10, NULL)
+NULL
+Warnings:
+Error	1108	Incorrect parameters to procedure 'des_encrypt'
+select des_encrypt("hello", NULL);
+des_encrypt("hello", NULL)
+NULL
+Warnings:
+Error	1108	Incorrect parameters to procedure 'des_encrypt'
+select des_decrypt("hello",10);
+des_decrypt("hello",10)
+hello
+select des_decrypt(NULL);
+des_decrypt(NULL)
+NULL
+Warnings:
+Error	1108	Incorrect parameters to procedure 'des_decrypt'
+select des_decrypt(NULL, 10);
+des_decrypt(NULL, 10)
+NULL
+Warnings:
+Error	1108	Incorrect parameters to procedure 'des_decrypt'
+select des_decrypt(NULL, NULL);
+des_decrypt(NULL, NULL)
+NULL
+Warnings:
+Error	1108	Incorrect parameters to procedure 'des_decrypt'
+select des_decrypt(10, NULL);
+des_decrypt(10, NULL)
+10
+select des_decrypt("hello", NULL);
+des_decrypt("hello", NULL)
+hello
 SET @a=des_decrypt(des_encrypt("hello"));
 flush des_key_file;
 select @a = des_decrypt(des_encrypt("hello"));
@@ -134,6 +188,8 @@ NULL
 select hex(des_decrypt(des_encrypt("hello","hidden")));
 hex(des_decrypt(des_encrypt("hello","hidden")))
 NULL
+Warnings:
+Error	1108	Incorrect parameters to procedure 'des_decrypt'
 explain extended select des_decrypt(des_encrypt("hello",4),'password2'), des_decrypt(des_encrypt("hello","hidden"));
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used

mysql-test/r/func_encrypt_nossl.result

@@ -0,0 +1,93 @@
+select des_encrypt("test", 'akeystr');
+des_encrypt("test", 'akeystr')
+NULL
+Warnings:
+Error	1289	The 'des_encrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+select des_encrypt("test", 1);
+des_encrypt("test", 1)
+NULL
+Warnings:
+Error	1289	The 'des_encrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+select des_encrypt("test", 9);
+des_encrypt("test", 9)
+NULL
+Warnings:
+Error	1289	The 'des_encrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+select des_encrypt("test", 100);
+des_encrypt("test", 100)
+NULL
+Warnings:
+Error	1289	The 'des_encrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+select des_encrypt("test", NULL);
+des_encrypt("test", NULL)
+NULL
+Warnings:
+Error	1289	The 'des_encrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+select des_decrypt("test", 'anotherkeystr');
+des_decrypt("test", 'anotherkeystr')
+NULL
+Warnings:
+Error	1289	The 'des_decrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+select des_decrypt(1, 1);
+des_decrypt(1, 1)
+NULL
+Warnings:
+Error	1289	The 'des_decrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+select des_decrypt(des_encrypt("test", 'thekey'));
+des_decrypt(des_encrypt("test", 'thekey'))
+NULL
+Warnings:
+Error	1289	The 'des_decrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+select hex(des_encrypt("hello")),des_decrypt(des_encrypt("hello"));
+hex(des_encrypt("hello"))	des_decrypt(des_encrypt("hello"))
+NULL	NULL
+Warnings:
+Error	1289	The 'des_encrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+Error	1289	The 'des_decrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+select des_decrypt(des_encrypt("hello",4));
+des_decrypt(des_encrypt("hello",4))
+NULL
+Warnings:
+Error	1289	The 'des_decrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+select des_decrypt(des_encrypt("hello",'test'),'test');
+des_decrypt(des_encrypt("hello",'test'),'test')
+NULL
+Warnings:
+Error	1289	The 'des_decrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+select hex(des_encrypt("hello")),hex(des_encrypt("hello",5)),hex(des_encrypt("hello",'default_password'));
+hex(des_encrypt("hello"))	hex(des_encrypt("hello",5))	hex(des_encrypt("hello",'default_password'))
+NULL	NULL	NULL
+Warnings:
+Error	1289	The 'des_encrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+Error	1289	The 'des_encrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+Error	1289	The 'des_encrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+select des_decrypt(des_encrypt("hello"),'default_password');
+des_decrypt(des_encrypt("hello"),'default_password')
+NULL
+Warnings:
+Error	1289	The 'des_decrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+select des_decrypt(des_encrypt("hello",4),'password4');
+des_decrypt(des_encrypt("hello",4),'password4')
+NULL
+Warnings:
+Error	1289	The 'des_decrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+SET @a=des_decrypt(des_encrypt("hello"));
+Warnings:
+Error	1289	The 'des_decrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+flush des_key_file;
+select @a = des_decrypt(des_encrypt("hello"));
+@a = des_decrypt(des_encrypt("hello"))
+NULL
+select hex("hello");
+hex("hello")
+68656C6C6F
+select hex(des_decrypt(des_encrypt("hello",4),'password2'));
+hex(des_decrypt(des_encrypt("hello",4),'password2'))
+NULL
+Warnings:
+Error	1289	The 'des_decrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working
+select hex(des_decrypt(des_encrypt("hello","hidden")));
+hex(des_decrypt(des_encrypt("hello","hidden")))
+NULL
+Warnings:
+Error	1289	The 'des_decrypt' feature is disabled; you need MySQL built with '--with-openssl' to have it working

mysql-test/r/not_openssl.require

@@ -0,0 +1,2 @@
+Variable_name	Value
+have_openssl	NO

mysql-test/t/func_encrypt.test

@@ -59,6 +59,22 @@ select hex(des_encrypt("hello")),hex(des_encrypt("hello",5)),hex(des_encrypt("he
 select des_decrypt(des_encrypt("hello"),'default_password');
 select des_decrypt(des_encrypt("hello",4),'password4');
 
+# Test use of invalid parameters
+select des_encrypt("hello",10);
+select des_encrypt(NULL);
+select des_encrypt(NULL, 10);
+select des_encrypt(NULL, NULL);
+select des_encrypt(10, NULL);
+select des_encrypt("hello", NULL);
+
+select des_decrypt("hello",10);
+select des_decrypt(NULL);
+select des_decrypt(NULL, 10);
+select des_decrypt(NULL, NULL);
+select des_decrypt(10, NULL);
+select des_decrypt("hello", NULL);
+
+
 # Test flush
 SET @a=des_decrypt(des_encrypt("hello"));
 flush des_key_file;

mysql-test/t/func_encrypt_nossl.test

@@ -0,0 +1,36 @@
+-- source include/not_openssl.inc
+
+#
+# Test output from des_encrypt and des_decrypt when server is
+# compiled without openssl suuport
+#
+select des_encrypt("test", 'akeystr');
+select des_encrypt("test", 1);
+select des_encrypt("test", 9);
+select des_encrypt("test", 100);
+select des_encrypt("test", NULL);
+select des_decrypt("test", 'anotherkeystr');
+select des_decrypt(1, 1);
+select des_decrypt(des_encrypt("test", 'thekey'));
+
+
+#
+# Test default keys
+#
+select hex(des_encrypt("hello")),des_decrypt(des_encrypt("hello"));
+select des_decrypt(des_encrypt("hello",4));
+select des_decrypt(des_encrypt("hello",'test'),'test');
+select hex(des_encrypt("hello")),hex(des_encrypt("hello",5)),hex(des_encrypt("hello",'default_password'));
+select des_decrypt(des_encrypt("hello"),'default_password');
+select des_decrypt(des_encrypt("hello",4),'password4');
+
+# Test flush
+SET @a=des_decrypt(des_encrypt("hello"));
+flush des_key_file;
+select @a = des_decrypt(des_encrypt("hello"));
+
+# Test usage of wrong password
+select hex("hello");
+select hex(des_decrypt(des_encrypt("hello",4),'password2'));
+select hex(des_decrypt(des_encrypt("hello","hidden")));
+

sql/item_strfunc.cc

@@ -373,6 +373,7 @@ String *Item_func_des_encrypt::val_str(String *str)
 {
   DBUG_ASSERT(fixed == 1);
 #ifdef HAVE_OPENSSL
+  uint code= ER_WRONG_PARAMETERS_TO_PROCEDURE;
   DES_cblock ivec;
   struct st_des_keyblock keyblock;
   struct st_des_keyschedule keyschedule;
@@ -381,7 +382,7 @@ String *Item_func_des_encrypt::val_str(String *str)
   String *res= args[0]->val_str(str);
 
   if ((null_value=args[0]->null_value))
-    return 0;
+    goto error;
   if ((res_length=res->length()) == 0)
     return &my_empty_string;
 
@@ -429,6 +430,7 @@ String *Item_func_des_encrypt::val_str(String *str)
 
   tail=  (8-(res_length) % 8);			// 1..8 marking extra length
   res_length+=tail;
+  code= ER_OUT_OF_RESOURCES;
   if (tail && res->append(append_str, tail) || tmp_value.alloc(res_length+1))
     goto error;
   (*res)[res_length-1]=tail;			// save extra length
@@ -446,6 +448,13 @@ String *Item_func_des_encrypt::val_str(String *str)
   return &tmp_value;
 
 error:
+  push_warning_printf(current_thd,MYSQL_ERROR::WARN_LEVEL_ERROR,
+                          code, ER(code),
+                          "des_encrypt");
+#else
+  push_warning_printf(current_thd,MYSQL_ERROR::WARN_LEVEL_ERROR,
+                      ER_FEATURE_DISABLED, ER(ER_FEATURE_DISABLED),
+                      "des_encrypt","--with-openssl");
 #endif	/* HAVE_OPENSSL */
   null_value=1;
   return 0;
@@ -456,6 +465,7 @@ String *Item_func_des_decrypt::val_str(String *str)
 {
   DBUG_ASSERT(fixed == 1);
 #ifdef HAVE_OPENSSL
+  uint code= ER_WRONG_PARAMETERS_TO_PROCEDURE;
   DES_key_schedule ks1, ks2, ks3;
   DES_cblock ivec;
   struct st_des_keyblock keyblock;
@@ -464,7 +474,7 @@ String *Item_func_des_decrypt::val_str(String *str)
   uint length=res->length(),tail;
 
   if ((null_value=args[0]->null_value))
-    return 0;
+    goto error;
   length=res->length();
   if (length < 9 || (length % 8) != 1 || !((*res)[0] & 128))
     return res;				// Skip decryption if not encrypted
@@ -495,6 +505,7 @@ String *Item_func_des_decrypt::val_str(String *str)
     DES_set_key_unchecked(&keyblock.key2,&keyschedule.ks2);
     DES_set_key_unchecked(&keyblock.key3,&keyschedule.ks3);
   }
+  code= ER_OUT_OF_RESOURCES;
   if (tmp_value.alloc(length-1))
     goto error;
 
@@ -508,11 +519,19 @@ String *Item_func_des_decrypt::val_str(String *str)
 		       &ivec, FALSE);
   /* Restore old length of key */
   if ((tail=(uint) (uchar) tmp_value[length-2]) > 8)
-    goto error;					// Wrong key
+    goto wrong_key;				     // Wrong key
   tmp_value.length(length-1-tail);
   return &tmp_value;
 
 error:
+  push_warning_printf(current_thd,MYSQL_ERROR::WARN_LEVEL_ERROR,
+                          code, ER(code),
+                          "des_decrypt");
+wrong_key:
+#else
+  push_warning_printf(current_thd,MYSQL_ERROR::WARN_LEVEL_ERROR,
+                      ER_FEATURE_DISABLED, ER(ER_FEATURE_DISABLED),
+                      "des_decrypt","--with-openssl");
 #endif	/* HAVE_OPENSSL */
   null_value=1;
   return 0;