From 6ff70926985c49a32bab682d5ef6e54dcca86782 Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Wed, 3 Jan 2007 17:26:45 -0800 Subject: [PATCH 1/4] Split out the hidden class from the ha_tina file (probably should have just deleted it...) but this is a bit better. --- storage/csv/CMakeLists.txt | 2 +- storage/csv/Makefile.am | 6 +- storage/csv/ha_tina.cc | 48 --------------- storage/csv/ha_tina.h | 44 +------------- storage/csv/transparent_file.cc | 104 ++++++++++++++++++++++++++++++++ storage/csv/transparent_file.h | 41 +++++++++++++ 6 files changed, 150 insertions(+), 95 deletions(-) create mode 100644 storage/csv/transparent_file.cc create mode 100644 storage/csv/transparent_file.h diff --git a/storage/csv/CMakeLists.txt b/storage/csv/CMakeLists.txt index 55e9b50fbfc..9088e2b7aa9 100644 --- a/storage/csv/CMakeLists.txt +++ b/storage/csv/CMakeLists.txt @@ -4,4 +4,4 @@ SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DSAFEMALLOC -DSAFE_MUTEX") INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/include ${CMAKE_SOURCE_DIR}/sql ${CMAKE_SOURCE_DIR}/regex ${CMAKE_SOURCE_DIR}/extra/yassl/include) -ADD_LIBRARY(csv ha_tina.cc ha_tina.h) +ADD_LIBRARY(csv ha_tina.cc ha_tina.h transparent_file.cc transparent_file.h) diff --git a/storage/csv/Makefile.am b/storage/csv/Makefile.am index e6bda138120..15ef5c71756 100644 --- a/storage/csv/Makefile.am +++ b/storage/csv/Makefile.am @@ -28,18 +28,18 @@ INCLUDES = -I$(top_builddir)/include \ LDADD = DEFS = @DEFS@ -noinst_HEADERS = ha_tina.h +noinst_HEADERS = ha_tina.h transparent_file.h EXTRA_LTLIBRARIES = ha_csv.la pkglib_LTLIBRARIES = @plugin_csv_shared_target@ ha_csv_la_LDFLAGS = -module -rpath $(MYSQLLIBdir) ha_csv_la_CXXFLAGS = $(AM_CFLAGS) -DMYSQL_PLUGIN -ha_csv_la_SOURCES = ha_tina.cc +ha_csv_la_SOURCES = transparent_file.cc ha_tina.cc EXTRA_LIBRARIES = libcsv.a noinst_LIBRARIES = @plugin_csv_static_target@ libcsv_a_CXXFLAGS = $(AM_CFLAGS) -libcsv_a_SOURCES = ha_tina.cc +libcsv_a_SOURCES = transparent_file.cc ha_tina.cc EXTRA_DIST = CMakeLists.txt plug.in # Don't update the files from bitkeeper diff --git a/storage/csv/ha_tina.cc b/storage/csv/ha_tina.cc index b957bcef3ea..712c2d9d29a 100644 --- a/storage/csv/ha_tina.cc +++ b/storage/csv/ha_tina.cc @@ -77,54 +77,6 @@ static handler *tina_create_handler(handlerton *hton, MEM_ROOT *mem_root); -off_t Transparent_file::read_next() -{ - off_t bytes_read; - - /* - No need to seek here, as the file managed by Transparent_file class - always points to upper_bound byte - */ - if ((bytes_read= my_read(filedes, buff, buff_size, MYF(0))) == MY_FILE_ERROR) - return -1; - - /* end of file */ - if (!bytes_read) - return -1; - - lower_bound= upper_bound; - upper_bound+= bytes_read; - - return lower_bound; -} - - -char Transparent_file::get_value(off_t offset) -{ - off_t bytes_read; - - /* check boundaries */ - if ((lower_bound <= offset) && (offset < upper_bound)) - return buff[offset - lower_bound]; - else - { - VOID(my_seek(filedes, offset, MY_SEEK_SET, MYF(0))); - /* read appropriate portion of the file */ - if ((bytes_read= my_read(filedes, buff, buff_size, - MYF(0))) == MY_FILE_ERROR) - return 0; - - lower_bound= offset; - upper_bound= lower_bound + bytes_read; - - /* end of file */ - if (upper_bound == offset) - return 0; - - return buff[0]; - } -} - /***************************************************************************** ** TINA tables *****************************************************************************/ diff --git a/storage/csv/ha_tina.h b/storage/csv/ha_tina.h index 5068468fedf..0c667237c0f 100644 --- a/storage/csv/ha_tina.h +++ b/storage/csv/ha_tina.h @@ -16,6 +16,7 @@ #include #include #include +#include "transparent_file.h" #define DEFAULT_CHAIN_LENGTH 512 /* @@ -55,49 +56,6 @@ struct tina_set { off_t end; }; -class Transparent_file -{ - File filedes; - byte *buff; /* in-memory window to the file or mmaped area */ - /* current window sizes */ - off_t lower_bound; - off_t upper_bound; - uint buff_size; - - public: - - Transparent_file() : lower_bound(0), buff_size(IO_SIZE) - { buff= (byte *) my_malloc(buff_size*sizeof(byte), MYF(MY_WME)); } - - ~Transparent_file() - { my_free((gptr)buff, MYF(MY_ALLOW_ZERO_PTR)); } - - void init_buff(File filedes_arg) - { - filedes= filedes_arg; - /* read the beginning of the file */ - lower_bound= 0; - VOID(my_seek(filedes, 0, MY_SEEK_SET, MYF(0))); - if (filedes && buff) - upper_bound= my_read(filedes, buff, buff_size, MYF(0)); - } - - byte *ptr() - { return buff; } - - off_t start() - { return lower_bound; } - - off_t end() - { return upper_bound; } - - /* get a char from the given position in the file */ - char get_value (off_t offset); - /* shift a buffer windows to see the next part of the file */ - off_t read_next(); - -}; - class ha_tina: public handler { THR_LOCK_DATA lock; /* MySQL lock */ diff --git a/storage/csv/transparent_file.cc b/storage/csv/transparent_file.cc new file mode 100644 index 00000000000..27cc8c024b4 --- /dev/null +++ b/storage/csv/transparent_file.cc @@ -0,0 +1,104 @@ +/* Copyright (C) 2003 MySQL AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#ifdef USE_PRAGMA_IMPLEMENTATION +#pragma implementation // gcc: Class implementation +#endif + +#include "mysql_priv.h" +#include "transparent_file.h" + +Transparent_file::Transparent_file() : lower_bound(0), buff_size(IO_SIZE) +{ + buff= (byte *) my_malloc(buff_size*sizeof(byte), MYF(MY_WME)); +} + +Transparent_file::~Transparent_file() +{ + my_free((gptr)buff, MYF(MY_ALLOW_ZERO_PTR)); +} + +void Transparent_file::init_buff(File filedes_arg) +{ + filedes= filedes_arg; + /* read the beginning of the file */ + lower_bound= 0; + VOID(my_seek(filedes, 0, MY_SEEK_SET, MYF(0))); + if (filedes && buff) + upper_bound= my_read(filedes, buff, buff_size, MYF(0)); +} + +byte *Transparent_file::ptr() +{ + return buff; +} + +off_t Transparent_file::start() +{ + return lower_bound; +} + +off_t Transparent_file::end() +{ + return upper_bound; +} + +off_t Transparent_file::read_next() +{ + off_t bytes_read; + + /* + No need to seek here, as the file managed by Transparent_file class + always points to upper_bound byte + */ + if ((bytes_read= my_read(filedes, buff, buff_size, MYF(0))) == MY_FILE_ERROR) + return -1; + + /* end of file */ + if (!bytes_read) + return -1; + + lower_bound= upper_bound; + upper_bound+= bytes_read; + + return lower_bound; +} + + +char Transparent_file::get_value(off_t offset) +{ + off_t bytes_read; + + /* check boundaries */ + if ((lower_bound <= offset) && (offset < upper_bound)) + return buff[offset - lower_bound]; + else + { + VOID(my_seek(filedes, offset, MY_SEEK_SET, MYF(0))); + /* read appropriate portion of the file */ + if ((bytes_read= my_read(filedes, buff, buff_size, + MYF(0))) == MY_FILE_ERROR) + return 0; + + lower_bound= offset; + upper_bound= lower_bound + bytes_read; + + /* end of file */ + if (upper_bound == offset) + return 0; + + return buff[0]; + } +} diff --git a/storage/csv/transparent_file.h b/storage/csv/transparent_file.h new file mode 100644 index 00000000000..ceb59ec7caf --- /dev/null +++ b/storage/csv/transparent_file.h @@ -0,0 +1,41 @@ +/* Copyright (C) 2003 MySQL AB + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +#include +#include +#include + + +class Transparent_file +{ + File filedes; + byte *buff; /* in-memory window to the file or mmaped area */ + /* current window sizes */ + off_t lower_bound; + off_t upper_bound; + uint buff_size; + +public: + + Transparent_file(); + ~Transparent_file(); + + void init_buff(File filedes_arg); + byte *ptr(); + off_t start(); + off_t end(); + char get_value (off_t offset); + off_t read_next(); +}; From 452a1331b27ede1e8ef9baf274708f5cd4258edf Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Thu, 4 Jan 2007 11:41:17 -0800 Subject: [PATCH 2/4] The CSV format has always relied on numbers being quoted, which doesn't always happen. This fixes that so that numbers can now be unquoted (and the output does this as well so that the log takes up less space). --- mysql-test/r/csv.result | 26 +++++-- mysql-test/t/csv.test | 16 +++- storage/csv/ha_tina.cc | 162 ++++++++++++++++++++++++---------------- 3 files changed, 134 insertions(+), 70 deletions(-) diff --git a/mysql-test/r/csv.result b/mysql-test/r/csv.result index 32ca47e20d2..6456eb1d1ba 100644 --- a/mysql-test/r/csv.result +++ b/mysql-test/r/csv.result @@ -5210,16 +5210,32 @@ create table bug22080_3 (id int,string varchar(64)) Engine=CSV; insert into bug22080_1 values(1,'string'); insert into bug22080_1 values(2,'string'); insert into bug22080_1 values(3,'string'); -"1","string" +1,"string" 2","string" -"3","string" +3,"string" check table bug22080_2; Table Op Msg_type Msg_text test.bug22080_2 check error Corrupt -"1","string" -"2",string" -"3","string" +1,"string" +2,"string" +3,"string" check table bug22080_3; Table Op Msg_type Msg_text test.bug22080_3 check error Corrupt drop tables bug22080_1,bug22080_2,bug22080_3; +create table float_test (id float,string varchar(64)) Engine=CSV; +insert into float_test values(1.0,'string'); +insert into float_test values(2.23,'serg.g'); +insert into float_test values(0.03,'string'); +insert into float_test values(0.19,'string'); +insert into float_test values(.67,'string'); +insert into float_test values(9.67,'string'); +select * from float_test; +id string +1 string +2.23 serg.g +0.03 string +0.19 string +0.67 string +9.67 string +drop table float_test; diff --git a/mysql-test/t/csv.test b/mysql-test/t/csv.test index 60d38394fc0..135969fe644 100644 --- a/mysql-test/t/csv.test +++ b/mysql-test/t/csv.test @@ -1595,7 +1595,7 @@ insert into bug22080_1 values(2,'string'); insert into bug22080_1 values(3,'string'); # Currupt the file as described in the bug report ---exec sed -e 's/"2"/2"/' $MYSQLTEST_VARDIR/master-data/test/bug22080_1.CSV > $MYSQLTEST_VARDIR/master-data/test/bug22080_2.CSV +--exec sed -e 's/2/2"/' $MYSQLTEST_VARDIR/master-data/test/bug22080_1.CSV > $MYSQLTEST_VARDIR/master-data/test/bug22080_2.CSV --exec sed -e 's/2","/2",/' $MYSQLTEST_VARDIR/master-data/test/bug22080_1.CSV > $MYSQLTEST_VARDIR/master-data/test/bug22080_3.CSV --exec cat $MYSQLTEST_VARDIR/master-data/test/bug22080_2.CSV @@ -1605,3 +1605,17 @@ check table bug22080_2; check table bug22080_3; drop tables bug22080_1,bug22080_2,bug22080_3; + +# +# Testing float type +# +create table float_test (id float,string varchar(64)) Engine=CSV; +insert into float_test values(1.0,'string'); +insert into float_test values(2.23,'serg.g'); +insert into float_test values(0.03,'string'); +insert into float_test values(0.19,'string'); +insert into float_test values(.67,'string'); +insert into float_test values(9.67,'string'); +select * from float_test; + +drop table float_test; diff --git a/storage/csv/ha_tina.cc b/storage/csv/ha_tina.cc index 712c2d9d29a..0818b915618 100644 --- a/storage/csv/ha_tina.cc +++ b/storage/csv/ha_tina.cc @@ -462,6 +462,7 @@ int ha_tina::encode_quote(byte *buf) my_bitmap_map *org_bitmap= dbug_tmp_use_all_columns(table, table->read_set); buffer.length(0); + for (Field **field=table->field ; *field ; field++) { const char *ptr; @@ -478,50 +479,58 @@ int ha_tina::encode_quote(byte *buf) buffer.append(STRING_WITH_LEN("\"\",")); continue; } - else + + (*field)->val_str(&attribute,&attribute); + + if ((*field)->str_needs_quotes()) { - (*field)->val_str(&attribute,&attribute); ptr= attribute.ptr(); end_ptr= attribute.length() + ptr; + + buffer.append('"'); + + while (ptr < end_ptr) + { + if (*ptr == '"') + { + buffer.append('\\'); + buffer.append('"'); + *ptr++; + } + else if (*ptr == '\r') + { + buffer.append('\\'); + buffer.append('r'); + *ptr++; + } + else if (*ptr == '\\') + { + buffer.append('\\'); + buffer.append('\\'); + *ptr++; + } + else if (*ptr == '\n') + { + buffer.append('\\'); + buffer.append('n'); + *ptr++; + } + else + buffer.append(*ptr++); + } + buffer.append('"'); } - - buffer.append('"'); - - while (ptr < end_ptr) + else { - if (*ptr == '"') - { - buffer.append('\\'); - buffer.append('"'); - *ptr++; - } - else if (*ptr == '\r') - { - buffer.append('\\'); - buffer.append('r'); - *ptr++; - } - else if (*ptr == '\\') - { - buffer.append('\\'); - buffer.append('\\'); - *ptr++; - } - else if (*ptr == '\n') - { - buffer.append('\\'); - buffer.append('n'); - *ptr++; - } - else - buffer.append(*ptr++); + buffer.append(attribute); } - buffer.append('"'); + buffer.append(','); } // Remove the comma, add a line feed buffer.length(buffer.length() - 1); buffer.append('\n'); + //buffer.replace(buffer.length(), 0, "\n", 1); dbug_tmp_restore_column_map(table->read_set, org_bitmap); @@ -601,47 +610,72 @@ int ha_tina::find_current_row(byte *buf) buffer.length(0); if (curr_offset < end_offset && file_buff->get_value(curr_offset) == '"') - curr_offset++; // Incrementpast the first quote - else - goto err; - for(;curr_offset < end_offset; curr_offset++) { - // Need to convert line feeds! - if (file_buff->get_value(curr_offset) == '"' && - ((file_buff->get_value(curr_offset + 1) == ',') || - (curr_offset == end_offset -1 ))) + curr_offset++; // Incrementpast the first quote + + for(;curr_offset < end_offset; curr_offset++) { - curr_offset+= 2; // Move past the , and the " - break; - } - if (file_buff->get_value(curr_offset) == '\\' && - curr_offset != (end_offset - 1)) - { - curr_offset++; - if (file_buff->get_value(curr_offset) == 'r') - buffer.append('\r'); - else if (file_buff->get_value(curr_offset) == 'n' ) - buffer.append('\n'); - else if ((file_buff->get_value(curr_offset) == '\\') || - (file_buff->get_value(curr_offset) == '"')) - buffer.append(file_buff->get_value(curr_offset)); - else /* This could only happed with an externally created file */ + // Need to convert line feeds! + if (file_buff->get_value(curr_offset) == '"' && + ((file_buff->get_value(curr_offset + 1) == ',') || + (curr_offset == end_offset -1 ))) { - buffer.append('\\'); + curr_offset+= 2; // Move past the , and the " + break; + } + if (file_buff->get_value(curr_offset) == '\\' && + curr_offset != (end_offset - 1)) + { + curr_offset++; + if (file_buff->get_value(curr_offset) == 'r') + buffer.append('\r'); + else if (file_buff->get_value(curr_offset) == 'n' ) + buffer.append('\n'); + else if ((file_buff->get_value(curr_offset) == '\\') || + (file_buff->get_value(curr_offset) == '"')) + buffer.append(file_buff->get_value(curr_offset)); + else /* This could only happed with an externally created file */ + { + buffer.append('\\'); + buffer.append(file_buff->get_value(curr_offset)); + } + } + else // ordinary symbol + { + /* + We are at final symbol and no last quote was found => + we are working with a damaged file. + */ + if (curr_offset == end_offset - 1) + goto err; buffer.append(file_buff->get_value(curr_offset)); } } - else // ordinary symbol + } + else if (my_isdigit(system_charset_info, + file_buff->get_value(curr_offset))) + { + for(;curr_offset < end_offset; curr_offset++) { - /* - We are at final symbol and no last quote was found => - we are working with a damaged file. - */ - if (curr_offset == end_offset - 1) + if (file_buff->get_value(curr_offset) == ',') + { + curr_offset+= 1; // Move past the , + break; + } + + if (my_isdigit(system_charset_info, file_buff->get_value(curr_offset))) + buffer.append(file_buff->get_value(curr_offset)); + else if (file_buff->get_value(curr_offset) == '.') + buffer.append(file_buff->get_value(curr_offset)); + else goto err; - buffer.append(file_buff->get_value(curr_offset)); } } + else + { + goto err; + } + if (bitmap_is_set(table->read_set, (*field)->field_index)) (*field)->store(buffer.ptr(), buffer.length(), system_charset_info); } From 77004aabf20f4bda7f9638c1aac868a77a40f055 Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Fri, 5 Jan 2007 20:20:53 -0800 Subject: [PATCH 3/4] Cleaning up warning for windows. Innodb is compiled but disable on Windows, so it should be fine. NDB wasn't compiled in, which is what caused the issue. --- mysql-test/r/warnings.result | 4 ++-- mysql-test/t/warnings.test | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/warnings.result b/mysql-test/r/warnings.result index 21c553951f6..93c83c57c45 100644 --- a/mysql-test/r/warnings.result +++ b/mysql-test/r/warnings.result @@ -166,10 +166,10 @@ show variables like 'max_error_count'; Variable_name Value max_error_count 10 drop table t1; -create table t1 (id int) engine=NDB; +create table t1 (id int) engine=Innodb; Warnings: Warning 1266 Using storage engine MyISAM for table 't1' -alter table t1 engine=NDB; +alter table t1 engine=Innodb; Warnings: Warning 1266 Using storage engine MyISAM for table 't1' drop table t1; diff --git a/mysql-test/t/warnings.test b/mysql-test/t/warnings.test index eb5a24a8604..ce246e5543b 100644 --- a/mysql-test/t/warnings.test +++ b/mysql-test/t/warnings.test @@ -113,8 +113,8 @@ show variables like 'max_error_count'; # Test for handler type # drop table t1; -create table t1 (id int) engine=NDB; -alter table t1 engine=NDB; +create table t1 (id int) engine=Innodb; +alter table t1 engine=Innodb; drop table t1; # From 05d05c776f106d2e7a255a183884d3bf64c8979a Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Fri, 5 Jan 2007 22:28:53 -0800 Subject: [PATCH 4/4] Fix for when shell version of mysql-test-run is called. --- mysql-test/t/warnings-master.opt | 1 + mysql-test/t/warnings.test | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/mysql-test/t/warnings-master.opt b/mysql-test/t/warnings-master.opt index e69de29bb2d..bea071a9c9b 100644 --- a/mysql-test/t/warnings-master.opt +++ b/mysql-test/t/warnings-master.opt @@ -0,0 +1 @@ +--skip-innodb diff --git a/mysql-test/t/warnings.test b/mysql-test/t/warnings.test index ce246e5543b..c9aba4549c2 100644 --- a/mysql-test/t/warnings.test +++ b/mysql-test/t/warnings.test @@ -150,8 +150,6 @@ drop table t1; # Bug#20778: strange characters in warning message 1366 when called in SP # -let $engine_type= innodb; - CREATE TABLE t1( f1 CHAR(20) ); CREATE TABLE t2( f1 CHAR(20), f2 CHAR(25) ); CREATE TABLE t3( f1 CHAR(20), f2 CHAR(25), f3 DATE );