From 5163c03bada8cc800499284e3da9d1e4ecdcf8f0 Mon Sep 17 00:00:00 2001 From: "joerg@mysql.com" <> Date: Wed, 27 Oct 2004 21:59:34 +0200 Subject: [PATCH 1/3] Fix for bug#5385 ("Text files have unix line break"): Windows style line end \r\n on README, license, and similar essential plaintext files. --- scripts/make_win_src_distribution.sh | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/scripts/make_win_src_distribution.sh b/scripts/make_win_src_distribution.sh index e58d44f336b..fd7884068ba 100644 --- a/scripts/make_win_src_distribution.sh +++ b/scripts/make_win_src_distribution.sh @@ -6,7 +6,6 @@ version=@VERSION@ export version -SOURCE=`pwd` CP="cp -p" DEBUG=0 @@ -24,6 +23,7 @@ if [ ! -f scripts/make_win_src_distribution ]; then echo "ERROR : You must run this script from the MySQL top-level directory" exit 1 fi +SOURCE=`pwd` # # Check for source compilation/configuration @@ -119,7 +119,7 @@ unix_to_dos() for arg do print_debug "Replacing LF -> CRLF from '$arg'" - cat $arg | awk '{sub(/$/,"\r");print}' > $arg.tmp + awk '{sub(/$/,"\r");print}' < $arg > $arg.tmp rm -f $arg mv $arg.tmp $arg done @@ -138,14 +138,6 @@ if [ -d $BASE ] ; then fi $CP -r $SOURCE/VC++Files $BASE -( -find $BASE \( -name "*.dsp" -o -name "*.dsw" \) -and -not -path \*SCCS\* -print -)|( - while read v - do - unix_to_dos $v - done -) # # Process version tags in InstallShield files @@ -281,7 +273,6 @@ for i in COPYING ChangeLog README EXCEPTIONS-CLIENT\ Docs/manual_toc.html Docs/manual.html \ Docs/manual.txt Docs/mysqld_error.txt \ Docs/INSTALL-BINARY Docs/internals.texi - do print_debug "Copying file '$i'" if [ -f $i ] @@ -322,7 +313,18 @@ done ./extra/replace std:: "" < $BASE/sql/sql_yacc.cpp | sed '/^ *switch (yytype)$/ { N; /\n *{$/ { N; /\n *default:$/ { N; /\n *break;$/ { N; /\n *}$/ d; };};};} ' > $BASE/sql/sql_yacc.cpp-new mv $BASE/sql/sql_yacc.cpp-new $BASE/sql/sql_yacc.cpp -unix_to_dos $BASE/README +# +# Search the tree for plain text files and adapt the line end marker +# +find $BASE \( -name "*.dsp" -o -name "*.dsw" -o -name "*.cnf" -o -name "*.ini" \ + -o -name COPYING -o -name ChangeLog -o -name EXCEPTIONS-CLIENT -o -name "INSTALL*" -o -name LICENSE -o -name "README*" \) -type f -print \ +| while read v + do + unix_to_dos $v + done +# File extension '.txt' matches too many other files, error messages etc. +unix_to_dos $BASE/Docs/*.txt + mv $BASE/README $BASE/README.txt # From 2fc0d7e422535882c94ddfe76ded8f357c2e3b13 Mon Sep 17 00:00:00 2001 From: "bar@mysql.com" <> Date: Thu, 28 Oct 2004 11:43:31 +0500 Subject: [PATCH 2/3] An extra "separator" class member was removed from Item_func_concat_ws. args[0] is now used instead. --- sql/item_func.cc | 4 +-- sql/item_strfunc.cc | 62 ++++++++------------------------------------- sql/item_strfunc.h | 20 +-------------- sql/sql_yacc.yy | 2 +- 4 files changed, 14 insertions(+), 74 deletions(-) diff --git a/sql/item_func.cc b/sql/item_func.cc index 17cf8642ce5..879c0b36bdd 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -2873,10 +2873,10 @@ void Item_func_match::init_search(bool no_order) if (key == NO_SUCH_KEY) { List fields; + fields.push_back(new Item_string(" ",1, cmp_collation.collation)); for (uint i=1; i < arg_count; i++) fields.push_back(args[i]); - concat=new Item_func_concat_ws(new Item_string(" ",1, - cmp_collation.collation), fields); + concat=new Item_func_concat_ws(fields); /* Above function used only to get value and do not need fix_fields for it: Item_string - basic constant diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc index d78a3ac88a4..c51894afde4 100644 --- a/sql/item_strfunc.cc +++ b/sql/item_strfunc.cc @@ -532,7 +532,7 @@ String *Item_func_concat_ws::val_str(String *str) uint i; null_value=0; - if (!(sep_str= separator->val_str(&tmp_sep_str))) + if (!(sep_str= args[0]->val_str(&tmp_sep_str))) goto null; use_as_buff= &tmp_value; @@ -541,7 +541,7 @@ String *Item_func_concat_ws::val_str(String *str) // Skip until non-null argument is found. // If not, return the empty string - for (i=0; i < arg_count; i++) + for (i=1; i < arg_count; i++) if ((res= args[i]->val_str(str))) break; if (i == arg_count) @@ -635,67 +635,25 @@ null: return 0; } -void Item_func_concat_ws::split_sum_func(THD *thd, Item **ref_pointer_array, - List &fields) -{ - if (separator->with_sum_func && separator->type() != SUM_FUNC_ITEM) - separator->split_sum_func(thd, ref_pointer_array, fields); - else if (separator->used_tables() || separator->type() == SUM_FUNC_ITEM) - { - uint el= fields.elements; - Item *new_item= new Item_ref(ref_pointer_array + el, 0, separator->name); - fields.push_front(separator); - ref_pointer_array[el]= separator; - thd->change_item_tree(&separator, new_item); - } - Item_str_func::split_sum_func(thd, ref_pointer_array, fields); -} void Item_func_concat_ws::fix_length_and_dec() { - collation.set(separator->collation); - max_length=separator->max_length*(arg_count-1); - for (uint i=0 ; i < arg_count ; i++) - { - DTCollation tmp(collation.collation, collation.derivation); + max_length=0; + + if (agg_arg_collations(collation, args, arg_count)) + return; + + max_length= arg_count > 1 ? args[0]->max_length * (arg_count - 2) : 0; + for (uint i=1 ; i < arg_count ; i++) max_length+=args[i]->max_length; - if (collation.aggregate(args[i]->collation)) - { - collation.set(tmp); // Restore the previous value - my_coll_agg_error(collation, args[i]->collation, func_name()); - break; - } - } + if (max_length > MAX_BLOB_WIDTH) { max_length=MAX_BLOB_WIDTH; maybe_null=1; } - used_tables_cache|= separator->used_tables(); - not_null_tables_cache&= separator->not_null_tables(); - const_item_cache&= separator->const_item(); - with_sum_func= with_sum_func || separator->with_sum_func; } -void Item_func_concat_ws::update_used_tables() -{ - Item_func::update_used_tables(); - separator->update_used_tables(); - used_tables_cache|=separator->used_tables(); - const_item_cache&=separator->const_item(); -} - -void Item_func_concat_ws::print(String *str) -{ - str->append("concat_ws(", 10); - separator->print(str); - if (arg_count) - { - str->append(','); - print_args(str, 0); - } - str->append(')'); -} String *Item_func_reverse::val_str(String *str) { diff --git a/sql/item_strfunc.h b/sql/item_strfunc.h index 77c1caec9fc..afe03c31345 100644 --- a/sql/item_strfunc.h +++ b/sql/item_strfunc.h @@ -89,30 +89,12 @@ public: class Item_func_concat_ws :public Item_str_func { - Item *separator; String tmp_value; - public: - Item_func_concat_ws(Item *a,List &list) - :Item_str_func(list),separator(a) {} + Item_func_concat_ws(List &list) :Item_str_func(list) {} String *val_str(String *); void fix_length_and_dec(); - void update_used_tables(); - bool fix_fields(THD *thd, TABLE_LIST *tlist, Item **ref) - { - DBUG_ASSERT(fixed == 0); - return (separator->fix_fields(thd, tlist, &separator) || - separator->check_cols(1) || - Item_func::fix_fields(thd, tlist, ref)); - } - void split_sum_func(THD *thd, Item **ref_pointer_array, List &fields); const char *func_name() const { return "concat_ws"; } - bool walk(Item_processor processor, byte *arg) - { - return separator->walk(processor, arg) || - Item_str_func::walk(processor, arg); - } - void print(String *str); }; class Item_func_reverse :public Item_str_func diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 142f52b87ab..4538cb6e6ac 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -2889,7 +2889,7 @@ simple_expr: | CONCAT '(' expr_list ')' { $$= new Item_func_concat(* $3); } | CONCAT_WS '(' expr ',' expr_list ')' - { $$= new Item_func_concat_ws($3, *$5); } + { $5->push_front($3); $$= new Item_func_concat_ws(*$5); } | CONVERT_TZ_SYM '(' expr ',' expr ',' expr ')' { Lex->time_zone_tables_used= &fake_time_zone_tables_list; From 541883f9d89a8d38affba60bf9506289a6232da1 Mon Sep 17 00:00:00 2001 From: "bar@mysql.com" <> Date: Thu, 28 Oct 2004 15:21:20 +0500 Subject: [PATCH 3/3] Produce a "truncated" warning if a value cannot be converted into the column character set on INSERT/UPDATE. --- mysql-test/r/alter_table.result | 1 + mysql-test/r/ctype_recoding.result | 8 ++++++++ mysql-test/r/fulltext.result | 5 +++++ mysql-test/t/alter_table.test | 1 + mysql-test/t/ctype_recoding.test | 10 ++++++++++ sql/field.cc | 32 ++++++++++++++++++++---------- sql/sql_string.cc | 21 ++++++++++++++++---- sql/sql_string.h | 4 ++-- 8 files changed, 66 insertions(+), 16 deletions(-) diff --git a/mysql-test/r/alter_table.result b/mysql-test/r/alter_table.result index e85ad303564..89b8bd66848 100644 --- a/mysql-test/r/alter_table.result +++ b/mysql-test/r/alter_table.result @@ -349,6 +349,7 @@ alter table t1 change a a char(10) character set koi8r; select a,hex(a) from t1; a hex(a) ÔÅÓÔ D4C5D3D4 +delete from t1; show create table t1; Table Create Table t1 CREATE TABLE `t1` ( diff --git a/mysql-test/r/ctype_recoding.result b/mysql-test/r/ctype_recoding.result index be792c007fc..72d19885101 100644 --- a/mysql-test/r/ctype_recoding.result +++ b/mysql-test/r/ctype_recoding.result @@ -166,3 +166,11 @@ ERROR HY000: Invalid utf8 character string: ' SET NAMES utf8; CREATE TABLE `goodÐÌÏÈÏ` (a int); ERROR HY000: Invalid utf8 character string: 'ÐÌÏÈÏ` (a int)' +set names latin1; +create table t1 (a char(10) character set koi8r, b text character set koi8r); +insert into t1 values ('test','test'); +insert into t1 values ('ÊÃÕË','ÊÃÕË'); +Warnings: +Warning 1265 Data truncated for column 'a' at row 1 +Warning 1265 Data truncated for column 'b' at row 1 +drop table t1; diff --git a/mysql-test/r/fulltext.result b/mysql-test/r/fulltext.result index 30c4c75f3d1..d07e4df6e73 100644 --- a/mysql-test/r/fulltext.result +++ b/mysql-test/r/fulltext.result @@ -356,6 +356,11 @@ SELECT t, collation(t),MATCH t AGAINST ('Osnabruck') FROM t1 WHERE MATCH t AGAIN t collation(t) MATCH t AGAINST ('Osnabruck') aus Osnabrück utf8_general_ci 1.591139793396 alter table t1 modify t varchar(200) collate latin1_german2_ci not null; +Warnings: +Warning 1265 Data truncated for column 't' at row 3 +Warning 1265 Data truncated for column 't' at row 4 +Warning 1265 Data truncated for column 't' at row 5 +Warning 1265 Data truncated for column 't' at row 6 SELECT t, collation(t) FROM t1 WHERE MATCH t AGAINST ('Osnabrück'); t collation(t) aus Osnabrück latin1_german2_ci diff --git a/mysql-test/t/alter_table.test b/mysql-test/t/alter_table.test index eb35aa90fe2..e46027ae8d9 100644 --- a/mysql-test/t/alter_table.test +++ b/mysql-test/t/alter_table.test @@ -207,6 +207,7 @@ alter table t1 change a a text character set cp1251; select a,hex(a) from t1; alter table t1 change a a char(10) character set koi8r; select a,hex(a) from t1; +delete from t1; # # Test ALTER TABLE .. CHARACTER SET .. diff --git a/mysql-test/t/ctype_recoding.test b/mysql-test/t/ctype_recoding.test index 82d0643b577..5f417352d95 100644 --- a/mysql-test/t/ctype_recoding.test +++ b/mysql-test/t/ctype_recoding.test @@ -121,3 +121,13 @@ CREATE TABLE `good SET NAMES utf8; --error 1300 CREATE TABLE `goodÐÌÏÈÏ` (a int); + + +# +# Test that we produce a warnign when conversion loses data. +# +set names latin1; +create table t1 (a char(10) character set koi8r, b text character set koi8r); +insert into t1 values ('test','test'); +insert into t1 values ('ÊÃÕË','ÊÃÕË'); +drop table t1; diff --git a/sql/field.cc b/sql/field.cc index 3e625f3a582..261494d2125 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -4275,9 +4275,12 @@ int Field_string::store(const char *from,uint length,CHARSET_INFO *cs) /* Convert character set if nesessary */ if (String::needs_conversion(length, cs, field_charset, ¬_used)) { - tmpstr.copy(from, length, cs, field_charset); + uint conv_errors; + tmpstr.copy(from, length, cs, field_charset, &conv_errors); from= tmpstr.ptr(); length= tmpstr.length(); + if (conv_errors) + error= 1; } /* @@ -4300,11 +4303,11 @@ int Field_string::store(const char *from,uint length,CHARSET_INFO *cs) from+= field_charset->cset->scan(field_charset, from, end, MY_SEQ_SPACES); if (from != end) - { - set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1); - error=1; - } + error= 1; } + if (error) + set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1); + return error; } @@ -4528,16 +4531,20 @@ int Field_varstring::store(const char *from,uint length,CHARSET_INFO *cs) /* Convert character set if nesessary */ if (String::needs_conversion(length, cs, field_charset, ¬_used)) { - tmpstr.copy(from, length, cs, field_charset); + uint conv_errors; + tmpstr.copy(from, length, cs, field_charset, &conv_errors); from= tmpstr.ptr(); length= tmpstr.length(); + if (conv_errors) + error= 1; } if (length > field_length) { length=field_length; - set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1); error= 1; } + if (error) + set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1); memcpy(ptr+HA_KEY_BLOB_LENGTH,from,length); int2store(ptr, length); return error; @@ -4865,6 +4872,7 @@ void Field_blob::put_length(char *pos, uint32 length) int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs) { + int error= 0; if (!length) { bzero(ptr,Field_blob::pack_length()); @@ -4881,9 +4889,12 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs) if ((was_conversion= String::needs_conversion(length, cs, field_charset, ¬_used))) { - tmpstr.copy(from, length, cs, field_charset); + uint conv_errors; + tmpstr.copy(from, length, cs, field_charset, &conv_errors); from= tmpstr.ptr(); length= tmpstr.length(); + if (conv_errors) + error= 1; } copy_length= max_data_length(); @@ -4897,8 +4908,7 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs) min(length, copy_length), copy_length); if (copy_length < length) - set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1); - + error= 1; Field_blob::store_length(copy_length); if (was_conversion || table->copy_blobs || copy_length <= MAX_FIELD_WIDTH) { // Must make a copy @@ -4910,6 +4920,8 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs) } bmove(ptr+packlength,(char*) &from,sizeof(char*)); } + if (error) + set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, ER_WARN_DATA_TRUNCATED, 1); return 0; } diff --git a/sql/sql_string.cc b/sql/sql_string.cc index 1044dc4e58e..30559496fde 100644 --- a/sql/sql_string.cc +++ b/sql/sql_string.cc @@ -331,19 +331,26 @@ bool String::set_or_copy_aligned(const char *str,uint32 arg_length, /* Copy with charset convertion */ bool String::copy(const char *str, uint32 arg_length, - CHARSET_INFO *from_cs, CHARSET_INFO *to_cs) + CHARSET_INFO *from_cs, CHARSET_INFO *to_cs, uint *errors) { uint32 offset; if (!needs_conversion(arg_length, from_cs, to_cs, &offset)) + { + if (errors) + *errors= 0; return copy(str, arg_length, to_cs); + } if ((from_cs == &my_charset_bin) && offset) + { + if (errors) + *errors= 0; return copy_aligned(str, arg_length, offset, to_cs); - + } uint32 new_length= to_cs->mbmaxlen*arg_length; if (alloc(new_length)) return TRUE; str_length=copy_and_convert((char*) Ptr, new_length, to_cs, - str, arg_length, from_cs); + str, arg_length, from_cs, errors); str_charset=to_cs; return FALSE; } @@ -769,7 +776,8 @@ String *copy_if_not_alloced(String *to,String *from,uint32 from_length) uint32 copy_and_convert(char *to, uint32 to_length, CHARSET_INFO *to_cs, - const char *from, uint32 from_length, CHARSET_INFO *from_cs) + const char *from, uint32 from_length, CHARSET_INFO *from_cs, + uint *errors) { int cnvres; my_wc_t wc; @@ -780,6 +788,7 @@ copy_and_convert(char *to, uint32 to_length, CHARSET_INFO *to_cs, const uchar *) = from_cs->cset->mb_wc; int (*wc_mb)(struct charset_info_st *, my_wc_t, uchar *s, uchar *e)= to_cs->cset->wc_mb; + uint error_count= 0; while (1) { @@ -788,6 +797,7 @@ copy_and_convert(char *to, uint32 to_length, CHARSET_INFO *to_cs, from+= cnvres; else if (cnvres == MY_CS_ILSEQ) { + error_count++; from++; wc= '?'; } @@ -799,12 +809,15 @@ outp: to+= cnvres; else if (cnvres == MY_CS_ILUNI && wc != '?') { + error_count++; wc= '?'; goto outp; } else break; } + if (errors) + *errors= error_count; return (uint32) (to - to_start); } diff --git a/sql/sql_string.h b/sql/sql_string.h index 2d368991159..de8f4af58d9 100644 --- a/sql/sql_string.h +++ b/sql/sql_string.h @@ -29,7 +29,7 @@ int sortcmp(const String *a,const String *b, CHARSET_INFO *cs); String *copy_if_not_alloced(String *a,String *b,uint32 arg_length); uint32 copy_and_convert(char *to, uint32 to_length, CHARSET_INFO *to_cs, const char *from, uint32 from_length, - CHARSET_INFO *from_cs); + CHARSET_INFO *from_cs, uint *errors= 0); class String { @@ -199,7 +199,7 @@ public: CHARSET_INFO *cs); bool set_or_copy_aligned(const char *s, uint32 arg_length, CHARSET_INFO *cs); bool copy(const char*s,uint32 arg_length, CHARSET_INFO *csfrom, - CHARSET_INFO *csto); + CHARSET_INFO *csto, uint *errors= 0); bool append(const String &s); bool append(const char *s); bool append(const char *s,uint32 arg_length);