mirror/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-01-16 20:12:31 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Many files:

Browse source 
Added the MAX_LENGTH_FOR_SORT_DATA option
filesort.cc:
  Added the MAX_LENGTH_FOR_SORT_DATA option
This commit is contained in:
igor@hundin.mysql.fi 2003-04-24 14:33:33 +03:00
parent 5610c4dea6
commit 80615fade2
15 changed files with 534 additions and 190 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

483
sql/filesort.cc
View file

@ -1,4 +1,5 @@
/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -51,6 +52,10 @@ static int merge_index(SORTPARAM *param,uchar *sort_buffer,
static bool save_index(SORTPARAM *param,uchar **sort_keys, uint count);
static uint sortlength(SORT_FIELD *sortorder, uint s_length,
bool *multi_byte_charset);
static SORT_ADDON_FIELD *get_addon_fields(THD *thd, Field **ptabfield,
uint sortlength, uint *plength);
static void unpack_addon_fields(struct st_sort_addon_field *addon_field,
byte *buff);
/*
Creates a set of pointers that can be used to read the rows
@ -82,16 +87,48 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
DBUG_PUSH(""); /* No DBUG here */
#endif
outfile= table->io_cache;
outfile= table->sort.io_cache;
my_b_clear(&tempfile);
my_b_clear(&buffpek_pointers);
buffpek=0;
sort_keys= (uchar **) NULL;
error= 1;
bzero((char*) &param,sizeof(param));
param.sort_length= sortlength(sortorder, s_length, &multi_byte_charset);
param.ref_length= table->file->ref_length;
param.sort_length= (sortlength(sortorder,s_length, &multi_byte_charset)+
param.ref_length);
param.addon_field= 0;
param.addon_length= 0;
if (!(table->tmp_table || table->fulltext_searched))
{
/*
Get the descriptors of all fields whose values are appended
to sorted fields and get its total length in param.spack_length.
*/
param.addon_field= get_addon_fields(thd, table->field,
param.sort_length,
&param.addon_length);
}
table->sort.addon_buf= 0;
table->sort.addon_length= param.addon_length;
table->sort.addon_field= param.addon_field;
table->sort.unpack= unpack_addon_fields;
if (param.addon_field)
{
param.res_length= param.addon_length;
if (!(table->sort.addon_buf= (byte *) my_malloc(param.addon_length,
MYF(MY_WME))))
goto err;
}
else
{
param.res_length= param.ref_length;
/*
The reference to the record is considered
as an additional sorted field
*/
param.sort_length+= param.ref_length;
}
param.rec_length= param.sort_length+param.addon_length;
param.max_rows= max_rows;
if (select && select->quick)
@ -115,7 +152,7 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
records=table->file->estimate_number_of_rows();
selected_records_file= 0;
}
if (param.sort_length == param.ref_length && records > param.max_rows)
if (param.rec_length == param.ref_length && records > param.max_rows)
records=param.max_rows; /* purecov: inspected */
if (multi_byte_charset &&
@ -127,9 +164,9 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
while (memavl >= min_sort_memory)
{
ulong old_memavl;
ulong keys= memavl/(param.sort_length+sizeof(char*));
ulong keys= memavl/(param.rec_length+sizeof(char*));
param.keys=(uint) min(records+1, keys);
if ((sort_keys= (uchar **) make_char_array(param.keys, param.sort_length,
if ((sort_keys= (uchar **) make_char_array(param.keys, param.rec_length,
MYF(0))))
break;
old_memavl=memavl;
@ -176,8 +213,8 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
Use also the space previously used by string pointers in sort_buffer
for temporary key storage.
*/
param.keys=((param.keys*(param.sort_length+sizeof(char*))) /
param.sort_length-1);
param.keys=((param.keys*(param.rec_length+sizeof(char*))) /
param.rec_length-1);
maxbuffer--; // Offset from 0
if (merge_many_buff(&param,(uchar*) sort_keys,buffpek,&maxbuffer,
&tempfile))
@ -356,8 +393,8 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select,
if (write_keys(param,sort_keys,idx,buffpek_pointers,tempfile))
DBUG_RETURN(HA_POS_ERROR);
idx=0;
if (param->ref_length == param->sort_length &&
my_b_tell(tempfile)/param->sort_length >= param->max_rows)
if (param->ref_length == param->rec_length &&
my_b_tell(tempfile)/param->rec_length >= param->max_rows)
{
/*
We are writing the result index file and have found all
@ -385,7 +422,7 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select,
write_keys(param,sort_keys,idx,buffpek_pointers,tempfile))
DBUG_RETURN(HA_POS_ERROR); /* purecov: inspected */
DBUG_RETURN(my_b_inited(tempfile) ?
(ha_rows) (my_b_tell(tempfile)/param->sort_length) :
(ha_rows) (my_b_tell(tempfile)/param->rec_length) :
idx);
} /* find_all_keys */
@ -394,29 +431,30 @@ static ha_rows find_all_keys(SORTPARAM *param, SQL_SELECT *select,
static int
write_keys(SORTPARAM *param, register uchar **sort_keys, uint count,
IO_CACHE *buffpek_pointers, IO_CACHE *tempfile)
IO_CACHE *buffpek_pointers, IO_CACHE *tempfile)
{
uint sort_length;
uint sort_length, rec_length;
uchar **end;
BUFFPEK buffpek;
DBUG_ENTER("write_keys");
sort_length=param->sort_length;
sort_length= param->sort_length;
rec_length= param->rec_length;
#ifdef MC68000
quicksort(sort_keys,count,sort_length);
#else
my_string_ptr_sort((gptr) sort_keys,(uint) count,sort_length);
my_string_ptr_sort((gptr) sort_keys, (uint) count, sort_length);
#endif
if (!my_b_inited(tempfile) &&
open_cached_file(tempfile,mysql_tmpdir,TEMP_PREFIX,DISK_BUFFER_SIZE,
MYF(MY_WME)))
goto err; /* purecov: inspected */
buffpek.file_pos=my_b_tell(tempfile);
open_cached_file(tempfile, mysql_tmpdir, TEMP_PREFIX, DISK_BUFFER_SIZE,
MYF(MY_WME)))
goto err; /* purecov: inspected */
buffpek.file_pos= my_b_tell(tempfile);
if ((ha_rows) count > param->max_rows)
count=(uint) param->max_rows; /* purecov: inspected */
count=(uint) param->max_rows; /* purecov: inspected */
buffpek.count=(ha_rows) count;
for (end=sort_keys+count ; sort_keys != end ; sort_keys++)
if (my_b_write(tempfile,(byte*) *sort_keys,(uint) sort_length))
if (my_b_write(tempfile, (byte*) *sort_keys, (uint) rec_length))
goto err;
if (my_b_write(buffpek_pointers, (byte*) &buffpek, sizeof(buffpek)))
goto err;
@ -505,10 +543,10 @@ static void make_sortkey(register SORTPARAM *param,
}
else
{
my_strnxfrm(cs,(uchar*)to,length,(const uchar*)res->ptr(),length);
bzero((char *)to+length,diff);
my_strnxfrm(cs,(uchar*)to,length,(const uchar*)res->ptr(),length);
bzero((char *)to+length,diff);
}
break;
break;
}
case INT_RESULT:
{
@ -577,29 +615,56 @@ static void make_sortkey(register SORTPARAM *param,
else
to+= sort_field->length;
}
memcpy((byte*) to,ref_pos,(size_s) param->ref_length);/* Save filepos last */
if (param->addon_field)
{
/*
Save field values appended to sorted fields.
First null bit indicators are appended then field values follow.
In this implementation we use fixed layout for field values -
the same for all records.
*/
SORT_ADDON_FIELD *addonf= param->addon_field;
uchar *nulls= to;
DBUG_ASSERT(addonf);
bzero((char *) nulls, addonf->offset);
to+= addonf->offset;
for ( ; (field= addonf->field) ; addonf++)
{
if (addonf->null_bit && field->is_null())
nulls[addonf->null_offset]|= addonf->null_bit;
else
field->pack((char *) to, field->ptr);
to+= addonf->length;
}
}
else
{
/* Save filepos last */
memcpy((byte*) to, ref_pos, (size_s) param->ref_length);
}
return;
}
static bool save_index(SORTPARAM *param, uchar **sort_keys, uint count)
{
uint offset,ref_length;
uint offset,res_length;
byte *to;
DBUG_ENTER("save_index");
my_string_ptr_sort((gptr) sort_keys,(uint) count,param->sort_length);
ref_length=param->ref_length;
offset=param->sort_length-ref_length;
my_string_ptr_sort((gptr) sort_keys, (uint) count, param->sort_length);
res_length= param->res_length;
offset= param->rec_length-res_length;
if ((ha_rows) count > param->max_rows)
count=(uint) param->max_rows;
if (!(to=param->sort_form->record_pointers=
(byte*) my_malloc(ref_length*count,MYF(MY_WME))))
DBUG_RETURN(1); /* purecov: inspected */
for (uchar **end=sort_keys+count ; sort_keys != end ; sort_keys++)
if (!(to= param->sort_form->sort.record_pointers=
(byte*) my_malloc(res_length*count, MYF(MY_WME))))
DBUG_RETURN(1); /* purecov: inspected */
for (uchar **end= sort_keys+count ; sort_keys != end ; sort_keys++)
{
memcpy(to,*sort_keys+offset,ref_length);
to+=ref_length;
memcpy(to, *sort_keys+offset, res_length);
to+= res_length;
}
DBUG_RETURN(0);
}
@ -654,7 +719,7 @@ int merge_many_buff(SORTPARAM *param, uchar *sort_buffer,
/* This returns (uint) -1 if something goes wrong */
uint read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
uint sort_length)
uint rec_length)
{
register uint count;
uint length;
@ -662,33 +727,35 @@ uint read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek,
if ((count=(uint) min((ha_rows) buffpek->max_keys,buffpek->count)))
{
if (my_pread(fromfile->file,(byte*) buffpek->base,
(length= sort_length*count),buffpek->file_pos,MYF_RW))
(length= rec_length*count),buffpek->file_pos,MYF_RW))
return((uint) -1); /* purecov: inspected */
buffpek->key=buffpek->base;
buffpek->file_pos+= length; /* New filepos */
buffpek->count-= count;
buffpek->mem_count= count;
}
return (count*sort_length);
return (count*rec_length);
} /* read_to_buffer */
/* Merge buffers to one buffer */
/*
Merge buffers to one buffer
*/
int merge_buffers(SORTPARAM *param, IO_CACHE *from_file,
IO_CACHE *to_file, uchar *sort_buffer,
BUFFPEK *lastbuff, BUFFPEK *Fb, BUFFPEK *Tb,
int flag)
IO_CACHE *to_file, uchar *sort_buffer,
BUFFPEK *lastbuff, BUFFPEK *Fb, BUFFPEK *Tb,
int flag)
{
int error;
uint sort_length,offset;
uint rec_length,sort_length,res_length,offset;
ulong maxcount;
ha_rows max_rows,org_max_rows;
my_off_t to_start_filepos;
uchar *strpos;
BUFFPEK *buffpek,**refpek;
QUEUE queue;
qsort2_cmp cmp;
qsort2_cmp cmp;
volatile bool *killed= &current_thd->killed;
bool not_killable;
DBUG_ENTER("merge_buffers");
@ -697,29 +764,32 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file,
if (param->not_killable)
{
killed= &not_killable;
not_killable=0;
not_killable= 0;
}
error=0;
offset=(sort_length=param->sort_length)-param->ref_length;
maxcount=(ulong) (param->keys/((uint) (Tb-Fb) +1));
to_start_filepos=my_b_tell(to_file);
strpos=(uchar*) sort_buffer;
org_max_rows=max_rows=param->max_rows;
rec_length= param->rec_length;
res_length= param->res_length;
sort_length= param->sort_length;
offset= rec_length-res_length;
maxcount= (ulong) (param->keys/((uint) (Tb-Fb) +1));
to_start_filepos= my_b_tell(to_file);
strpos= (uchar*) sort_buffer;
org_max_rows=max_rows= param->max_rows;
if (init_queue(&queue,(uint) (Tb-Fb)+1,offsetof(BUFFPEK,key),0,
(queue_compare)
(cmp=get_ptr_compare(sort_length)),(void*) &sort_length))
DBUG_RETURN(1); /* purecov: inspected */
if (init_queue(&queue, (uint) (Tb-Fb)+1, offsetof(BUFFPEK,key), 0,
(queue_compare) (cmp= get_ptr_compare(sort_length)),
(void*) &sort_length))
DBUG_RETURN(1); /* purecov: inspected */
for (buffpek= Fb ; buffpek <= Tb ; buffpek++)
{
buffpek->base= strpos;
buffpek->max_keys=maxcount;
strpos+= (uint) (error=(int) read_to_buffer(from_file,buffpek,
sort_length));
buffpek->max_keys= maxcount;
strpos+= (uint) (error= (int) read_to_buffer(from_file, buffpek,
rec_length));
if (error == -1)
goto err; /* purecov: inspected */
queue_insert(&queue,(byte*) buffpek);
goto err; /* purecov: inspected */
queue_insert(&queue, (byte*) buffpek);
}
if (param->unique_buff)
@ -732,98 +802,101 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file,
This is safe as we know that there is always more than one element
in each block to merge (This is guaranteed by the Unique:: algorithm
*/
buffpek=(BUFFPEK*) queue_top(&queue);
memcpy(param->unique_buff, buffpek->key, sort_length);
if (my_b_write(to_file,(byte*) buffpek->key, sort_length))
buffpek= (BUFFPEK*) queue_top(&queue);
memcpy(param->unique_buff, buffpek->key, rec_length);
if (my_b_write(to_file, (byte*) buffpek->key, rec_length))
{
error=1; goto err; /* purecov: inspected */
error=1; goto err; /* purecov: inspected */
}
buffpek->key+=sort_length;
buffpek->key+= rec_length;
buffpek->mem_count--;
if (!--max_rows)
{
error=0; /* purecov: inspected */
goto end; /* purecov: inspected */
error= 0; /* purecov: inspected */
goto end; /* purecov: inspected */
}
queue_replaced(&queue); // Top element has been used
queue_replaced(&queue); // Top element has been used
}
else
cmp=0; // Not unique
cmp= 0; // Not unique
while (queue.elements > 1)
{
if (*killed)
{
error=1; goto err; /* purecov: inspected */
error= 1; goto err; /* purecov: inspected */
}
for (;;)
{
buffpek=(BUFFPEK*) queue_top(&queue);
if (cmp) // Remove duplicates
buffpek= (BUFFPEK*) queue_top(&queue);
if (cmp) // Remove duplicates
{
if (!(*cmp)(&sort_length, &(param->unique_buff),
(uchar**) &buffpek->key))
goto skip_duplicate;
memcpy(param->unique_buff, (uchar*) buffpek->key,sort_length);
if (!(*cmp)(&sort_length, &(param->unique_buff),
(uchar**) &buffpek->key))
goto skip_duplicate;
memcpy(param->unique_buff, (uchar*) buffpek->key, rec_length);
}
if (flag == 0)
{
if (my_b_write(to_file,(byte*) buffpek->key, sort_length))
{
error=1; goto err; /* purecov: inspected */
}
if (my_b_write(to_file,(byte*) buffpek->key, rec_length))
{
error=1; goto err; /* purecov: inspected */
}
}
else
{
WRITE_REF(to_file,(byte*) buffpek->key+offset);
if (my_b_write(to_file, (byte*) buffpek->key+offset, res_length))
{
error=1; goto err; /* purecov: inspected */
}
}
if (!--max_rows)
{
error=0; /* purecov: inspected */
goto end; /* purecov: inspected */
error= 0; /* purecov: inspected */
goto end; /* purecov: inspected */
}
skip_duplicate:
buffpek->key+=sort_length;
buffpek->key+= rec_length;
if (! --buffpek->mem_count)
{
if (!(error=(int) read_to_buffer(from_file,buffpek,
sort_length)))
{
uchar *base=buffpek->base;
ulong max_keys=buffpek->max_keys;
if (!(error= (int) read_to_buffer(from_file,buffpek,
rec_length)))
{
uchar *base= buffpek->base;
ulong max_keys= buffpek->max_keys;
VOID(queue_remove(&queue,0));
VOID(queue_remove(&queue,0));
/* Put room used by buffer to use in other buffer */
for (refpek= (BUFFPEK**) &queue_top(&queue);
refpek <= (BUFFPEK**) &queue_end(&queue);
refpek++)
{
buffpek= *refpek;
if (buffpek->base+buffpek->max_keys*sort_length == base)
{
buffpek->max_keys+=max_keys;
break;
}
else if (base+max_keys*sort_length == buffpek->base)
{
buffpek->base=base;
buffpek->max_keys+=max_keys;
break;
}
}
break; /* One buffer have been removed */
}
else if (error == -1)
goto err; /* purecov: inspected */
/* Put room used by buffer to use in other buffer */
for (refpek= (BUFFPEK**) &queue_top(&queue);
refpek <= (BUFFPEK**) &queue_end(&queue);
refpek++)
{
buffpek= *refpek;
if (buffpek->base+buffpek->max_keys*rec_length == base)
{
buffpek->max_keys+= max_keys;
break;
}
else if (base+max_keys*rec_length == buffpek->base)
{
buffpek->base= base;
buffpek->max_keys+= max_keys;
break;
}
}
break; /* One buffer have been removed */
}
else if (error == -1)
goto err; /* purecov: inspected */
}
queue_replaced(&queue); /* Top element has been replaced */
queue_replaced(&queue); /* Top element has been replaced */
}
}
buffpek=(BUFFPEK*) queue_top(&queue);
buffpek= (BUFFPEK*) queue_top(&queue);
buffpek->base= sort_buffer;
buffpek->max_keys=param->keys;
buffpek->max_keys= param->keys;
/*
As we know all entries in the buffer are unique, we only have to
@ -833,7 +906,7 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file,
{
if (!(*cmp)(&sort_length, &(param->unique_buff), (uchar**) &buffpek->key))
{
buffpek->key+=sort_length; // Remove duplicate
buffpek->key+= rec_length; // Remove duplicate
--buffpek->mem_count;
}
}
@ -841,37 +914,40 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file,
do
{
if ((ha_rows) buffpek->mem_count > max_rows)
{ /* Don't write too many records */
buffpek->mem_count=(uint) max_rows;
buffpek->count=0; /* Don't read more */
{ /* Don't write too many records */
buffpek->mem_count= (uint) max_rows;
buffpek->count= 0; /* Don't read more */
}
max_rows-=buffpek->mem_count;
max_rows-= buffpek->mem_count;
if (flag == 0)
{
if (my_b_write(to_file,(byte*) buffpek->key,
(sort_length*buffpek->mem_count)))
(rec_length*buffpek->mem_count)))
{
error=1; goto err; /* purecov: inspected */
error= 1; goto err; /* purecov: inspected */
}
}
else
{
register uchar *end;
strpos= buffpek->key+offset;
for (end=strpos+buffpek->mem_count*sort_length;
strpos != end ;
strpos+=sort_length)
{
WRITE_REF(to_file,strpos);
for (end= strpos+buffpek->mem_count*rec_length ;
strpos != end ;
strpos+= rec_length)
{
if (my_b_write(to_file, (byte *) strpos, res_length))
{
error=1; goto err;
}
}
}
}
while ((error=(int) read_to_buffer(from_file,buffpek,sort_length))
!= -1 && error != 0);
while ((error=(int) read_to_buffer(from_file,buffpek, rec_length))
!= -1 && error != 0);
end:
lastbuff->count=min(org_max_rows-max_rows,param->max_rows);
lastbuff->file_pos=to_start_filepos;
lastbuff->count= min(org_max_rows-max_rows, param->max_rows);
lastbuff->file_pos= to_start_filepos;
err:
delete_queue(&queue);
DBUG_RETURN(error);
@ -925,7 +1001,6 @@ sortlength(SORT_FIELD *sortorder, uint s_length, bool *multi_byte_charset)
sortorder->need_strxnfrm= 0;
if (sortorder->field)
{
if (sortorder->field->type() == FIELD_TYPE_BLOB)
sortorder->length= thd->variables.max_sort_length;
else
@ -947,7 +1022,7 @@ sortlength(SORT_FIELD *sortorder, uint s_length, bool *multi_byte_charset)
case STRING_RESULT:
sortorder->length=sortorder->item->max_length;
if (use_strnxfrm((cs=sortorder->item->charset())))
{
{
sortorder->length= sortorder->length*cs->strxfrm_multiply;
sortorder->need_strxnfrm= 1;
*multi_byte_charset= 1;
@ -981,6 +1056,148 @@ sortlength(SORT_FIELD *sortorder, uint s_length, bool *multi_byte_charset)
}
/*
Get descriptors of fields appended to sorted fields and
calculate its total length
SYNOPSIS
get_addon_fields()
thd Current thread
ptabfields Array of references to the table fields
sortlength Total length of sorted fields
plength out: Total length of appended fields
DESCRIPTION
The function first finds out what fields are used in the result set.
Then it calculates the length of the buffer to store the values of
these fields together with the value of sort values.
If the calculated length is not greater than max_length_for_sort_data
the function allocates memory for an array of descriptors containing
layouts for the values of the non-sorted fields in the buffer and
fills them.
NOTES
The null bits for the appended values are supposed to be put together
and stored the buffer just ahead of the value of the first field.
RETURN
Pointer to the layout descriptors for the appended fields, if any
NULL - if we do not store field values with sort data.
*/
static SORT_ADDON_FIELD *
get_addon_fields(THD *thd, Field **ptabfield, uint sortlength, uint *plength)
{
Field **pfield;
Field *field;
SORT_ADDON_FIELD *addonf;
uint length= 0;
uint fields= 0;
uint null_fields= 0;
/*
If there is a reference to a field in the query add it
to the the set of appended fields.
Note for future refinement:
This this a too strong condition.
Actually we need only the fields referred in the
result set. And for some of them it makes sense to use
the values directly from sorted fields.
*/
*plength= 0;
/*
The following statement is added to avoid sorting in alter_table.
The fact is the filter 'field->query_id != thd->query_id'
doesn't work for alter table
*/
if (thd->lex.sql_command != SQLCOM_SELECT)
return 0;
for (pfield= ptabfield; (field= *pfield) ; pfield++)
{
if (field->query_id != thd->query_id)
continue;
if (field->flags & BLOB_FLAG)
return 0;
length+= field->max_packed_col_length(field->pack_length());
if (field->maybe_null())
null_fields++;
fields++;
}
if (!fields)
return 0;
length+= (null_fields+7)/8;
if (length+sortlength > thd->variables.max_length_for_sort_data ||
!(addonf= (SORT_ADDON_FIELD *) my_malloc(sizeof(SORT_ADDON_FIELD)*
(fields+1), MYF(MY_WME))))
return 0;
*plength= length;
length= (null_fields+7)/8;
null_fields= 0;
for (pfield= ptabfield; (field= *pfield) ; pfield++)
{
if (field->query_id != thd->query_id)
continue;
addonf->field= field;
addonf->offset= length;
if (field->maybe_null())
{
addonf->null_offset= null_fields/8;
addonf->null_bit= 1<<(null_fields & 7);
null_fields++;
}
else
{
addonf->null_offset= 0;
addonf->null_bit= 0;
}
addonf->length= field->max_packed_col_length(field->pack_length());
length+= addonf->length;
addonf++;
}
addonf->field= 0; // Put end marker
DBUG_PRINT("info",("addon_length: %d",length));
return (addonf-fields);
}
/*
Copy (unpack) values appended to sorted fields from a buffer back to
their regular positions specified by the Field::ptr pointers.
SYNOPSIS
unpack_addon_fields()
addon_field Array of descriptors for appended fields
buff Buffer which to unpack the value from
NOTES
The function is supposed to be used only as a callback function
when getting field values for the sorted result set.
RETURN
void.
*/
static void
unpack_addon_fields(struct st_sort_addon_field *addon_field, byte *buff)
{
Field *field;
SORT_ADDON_FIELD *addonf= addon_field;
for ( ; (field= addonf->field) ; addonf++)
{
if (addonf->null_bit && (addonf->null_bit & buff[addonf->null_offset]))
{
field->set_null();
continue;
}
field->set_notnull();
field->unpack(field->ptr, (char *) buff+addonf->offset);
}
}
/*
** functions to change a double or float to a sortable string
** The following should work for IEEE

7
sql/mysqld.cc
View file

@ -3457,7 +3457,7 @@ enum options
OPT_MAX_BINLOG_CACHE_SIZE, OPT_MAX_BINLOG_SIZE,
OPT_MAX_CONNECTIONS, OPT_MAX_CONNECT_ERRORS,
OPT_MAX_DELAYED_THREADS, OPT_MAX_HEP_TABLE_SIZE,
OPT_MAX_JOIN_SIZE, OPT_MAX_SORT_LENGTH,
OPT_MAX_JOIN_SIZE, OPT_MAX_LENGTH_FOR_SORT_DATA, OPT_MAX_SORT_LENGTH,
OPT_MAX_TMP_TABLES, OPT_MAX_USER_CONNECTIONS,
OPT_MAX_WRITE_LOCK_COUNT, OPT_BULK_INSERT_BUFFER_SIZE,
OPT_MAX_ERROR_COUNT, OPT_MAX_PREP_STMT,
@ -4159,6 +4159,11 @@ struct my_option my_long_options[] =
(gptr*) &global_system_variables.max_join_size,
(gptr*) &max_system_variables.max_join_size, 0, GET_HA_ROWS, REQUIRED_ARG,
~0L, 1, ~0L, 0, 1, 0},
{"max_length_for_sort_data", OPT_MAX_LENGTH_FOR_SORT_DATA,
"Max number of bytes in sorted records",
(gptr*) &global_system_variables.max_length_for_sort_data,
(gptr*) &max_system_variables.max_length_for_sort_data, 0, GET_ULONG,
REQUIRED_ARG, 1024, 4, 8192*1024L, 0, 1, 0},
{"max_prepared_statements", OPT_MAX_PREP_STMT,
"Max number of prepared_statements for a thread",
(gptr*) &global_system_variables.max_prep_stmt_count,

8
sql/opt_range.cc
View file

@ -349,13 +349,13 @@ SQL_SELECT *make_select(TABLE *head, table_map const_tables,
select->head=head;
select->cond=conds;
if (head->io_cache)
if (head->sort.io_cache)
{
select->file= *head->io_cache;
select->file= *head->sort.io_cache;
select->records=(ha_rows) (select->file.end_of_file/
head->file->ref_length);
my_free((gptr) (head->io_cache),MYF(0));
head->io_cache=0;
my_free((gptr) (head->sort.io_cache),MYF(0));
head->sort.io_cache=0;
}
DBUG_RETURN(select);
}

101
sql/records.cc
View file

@ -1,3 +1,4 @@
/* Copyright (C) 2000 MySQL AB & MySQL Finland AB & TCX DataKonsult AB
This program is free software; you can redistribute it and/or modify
@ -22,6 +23,8 @@
static int rr_quick(READ_RECORD *info);
static int rr_sequential(READ_RECORD *info);
static int rr_from_tempfile(READ_RECORD *info);
static int rr_unpack_from_tempfile(READ_RECORD *info);
static int rr_unpack_from_buffer(READ_RECORD *info);
static int rr_from_pointers(READ_RECORD *info);
static int rr_from_cache(READ_RECORD *info);
static int init_rr_cache(READ_RECORD *info);
@ -41,8 +44,16 @@ void init_read_record(READ_RECORD *info,THD *thd, TABLE *table,
info->table=table;
info->file= table->file;
info->forms= &info->table; /* Only one table */
info->record=table->record[0];
info->ref_length=table->file->ref_length;
if (table->sort.addon_field)
{
info->rec_buf= table->sort.addon_buf;
info->ref_length= table->sort.addon_length;
}
else
{
info->record= table->record[0];
info->ref_length= table->file->ref_length;
}
info->select=select;
info->print_error=print_error;
info->ignore_not_found_rows= 0;
@ -51,11 +62,12 @@ void init_read_record(READ_RECORD *info,THD *thd, TABLE *table,
if (select && my_b_inited(&select->file))
tempfile= &select->file;
else
tempfile= table->io_cache;
tempfile= table->sort.io_cache;
if (tempfile && my_b_inited(tempfile)) // Test if ref-records was used
{
DBUG_PRINT("info",("using rr_from_tempfile"));
info->read_record=rr_from_tempfile;
info->read_record= (table->sort.addon_field ?
rr_unpack_from_tempfile : rr_from_tempfile);
info->io_cache=tempfile;
reinit_io_cache(info->io_cache,READ_CACHE,0L,0,0);
info->ref_pos=table->file->ref;
@ -85,13 +97,15 @@ void init_read_record(READ_RECORD *info,THD *thd, TABLE *table,
DBUG_PRINT("info",("using rr_quick"));
info->read_record=rr_quick;
}
else if (table->record_pointers)
else if (table->sort.record_pointers)
{
DBUG_PRINT("info",("using record_pointers"));
table->file->rnd_init(0);
info->cache_pos=table->record_pointers;
info->cache_end=info->cache_pos+ table->found_records*info->ref_length;
info->read_record= rr_from_pointers;
info->cache_pos=table->sort.record_pointers;
info->cache_end=info->cache_pos+
table->sort.found_records*info->ref_length;
info->read_record= (table->sort.addon_field ?
rr_unpack_from_buffer : rr_from_pointers);
}
else
{
@ -112,7 +126,7 @@ void init_read_record(READ_RECORD *info,THD *thd, TABLE *table,
void end_read_record(READ_RECORD *info)
{ /* free cache if used */
{ /* free cache if used */
if (info->cache)
{
my_free_lock((char*) info->cache,MYF(0));
@ -120,6 +134,19 @@ void end_read_record(READ_RECORD *info)
}
if (info->table)
{
TABLE *table= info->table;
if (table->sort.record_pointers)
{
my_free((gptr) table->sort.record_pointers,MYF(0));
table->sort.record_pointers=0;
}
if (table->sort.addon_buf)
{
my_free((char *) table->sort.addon_buf, MYF(0));
my_free((char *) table->sort.addon_field, MYF(MY_ALLOW_ZERO_PTR));
table->sort.addon_buf=0;
table->sort.addon_field=0;
}
(void) info->file->extra(HA_EXTRA_NO_CACHE);
(void) info->file->rnd_end();
info->table=0;
@ -200,6 +227,34 @@ tryNext:
} /* rr_from_tempfile */
/*
Read a result set record from a temporary file after sorting
SYNOPSIS
rr_unpack_from_tempfile()
info Reference to the context including record descriptors
DESCRIPTION
The function first reads the next sorted record from the temporary file.
into a buffer. If a success it calls a callback function that unpacks
the fields values use in the result set from this buffer into their
positions in the regular record buffer.
RETURN
0 - Record successfully read.
-1 - There is no record to be read anymore.
*/
static int rr_unpack_from_tempfile(READ_RECORD *info)
{
if (my_b_read(info->io_cache, info->rec_buf, info->ref_length))
return -1;
TABLE *table= info->table;
(*table->sort.unpack)(table->sort.addon_field, info->rec_buf);
return 0;
}
static int rr_from_pointers(READ_RECORD *info)
{
int tmp;
@ -228,6 +283,34 @@ tryNext:
return tmp;
}
/*
Read a result set record from a buffer after sorting
SYNOPSIS
rr_unpack_from_buffer()
info Reference to the context including record descriptors
DESCRIPTION
The function first reads the next sorted record from the sort buffer.
If a success it calls a callback function that unpacks
the fields values use in the result set from this buffer into their
positions in the regular record buffer.
RETURN
0 - Record successfully read.
-1 - There is no record to be read anymore.
*/
static int rr_unpack_from_buffer(READ_RECORD *info)
{
if (info->cache_pos == info->cache_end)
return -1; /* End of buffer */
TABLE *table= info->table;
(*table->sort.unpack)(table->sort.addon_field, info->cache_pos);
info->cache_pos+= info->ref_length;
return 0;
}
/* cacheing of records from a database */
static int init_rr_cache(READ_RECORD *info)

6
sql/set_var.cc
View file

@ -166,6 +166,8 @@ sys_var_thd_ulong sys_pseudo_thread_id("pseudo_thread_id",
sys_var_thd_ha_rows sys_max_join_size("max_join_size",
&SV::max_join_size,
fix_max_join_size);
sys_var_thd_ulong sys_max_length_for_sort_data("max_length_for_sort_data",
&SV::max_length_for_sort_data);
#ifndef TO_BE_DELETED /* Alias for max_join_size */
sys_var_thd_ha_rows sys_sql_max_join_size("sql_max_join_size",
&SV::max_join_size,
@ -380,6 +382,7 @@ sys_var *sys_variables[]=
&sys_max_error_count,
&sys_max_heap_table_size,
&sys_max_join_size,
&sys_max_length_for_sort_data,
&sys_max_prep_stmt_count,
&sys_max_sort_length,
&sys_max_tmp_tables,
@ -533,6 +536,9 @@ struct show_var_st init_vars[]= {
{sys_max_delayed_threads.name,(char*) &sys_max_delayed_threads, SHOW_SYS},
{sys_max_heap_table_size.name,(char*) &sys_max_heap_table_size, SHOW_SYS},
{sys_max_join_size.name, (char*) &sys_max_join_size, SHOW_SYS},
{sys_max_length_for_sort_data.name,
(char*) &sys_max_length_for_sort_data,
SHOW_SYS},
{sys_max_prep_stmt_count.name,(char*) &sys_max_prep_stmt_count, SHOW_SYS},
{sys_max_sort_length.name, (char*) &sys_max_sort_length, SHOW_SYS},
{sys_max_user_connections.name,(char*) &sys_max_user_connections, SHOW_SYS},

13
sql/sql_base.cc
View file

@ -245,16 +245,11 @@ static void free_cache_entry(TABLE *table)
void free_io_cache(TABLE *table)
{
DBUG_ENTER("free_io_cache");
if (table->io_cache)
if (table->sort.io_cache)
{
close_cached_file(table->io_cache);
my_free((gptr) table->io_cache,MYF(0));
table->io_cache=0;
}
if (table->record_pointers)
{
my_free((gptr) table->record_pointers,MYF(0));
table->record_pointers=0;
close_cached_file(table->sort.io_cache);
my_free((gptr) table->sort.io_cache,MYF(0));
table->sort.io_cache=0;
}
DBUG_VOID_RETURN;
}

1
sql/sql_class.h
View file

@ -352,6 +352,7 @@ struct system_variables
ulong max_allowed_packet;
ulong max_error_count;
ulong max_heap_table_size;
ulong max_length_for_sort_data;
ulong max_prep_stmt_count;
ulong max_sort_length;
ulong max_tmp_tables;

10
sql/sql_delete.cc
View file

@ -124,13 +124,13 @@ int mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, ORDER *order,
bzero((char*) &tables,sizeof(tables));
tables.table = table;
table->io_cache = (IO_CACHE *) my_malloc(sizeof(IO_CACHE),
MYF(MY_FAE | MY_ZEROFILL));
table->sort.io_cache = (IO_CACHE *) my_malloc(sizeof(IO_CACHE),
MYF(MY_FAE | MY_ZEROFILL));
if (setup_order(thd, 0, &tables, fields, all_fields, order) ||
!(sortorder=make_unireg_sortorder(order, &length)) ||
(table->found_records = filesort(thd, table, sortorder, length,
(SQL_SELECT *) 0, HA_POS_ERROR,
&examined_rows))
(table->sort.found_records = filesort(thd, table, sortorder, length,
(SQL_SELECT *) 0, HA_POS_ERROR,
&examined_rows))
== HA_POS_ERROR)
{
delete select;

22
sql/sql_select.cc
View file

@ -4009,7 +4009,7 @@ Field *create_tmp_field(THD *thd, TABLE *table,Item *item, Item::Type type,
case Item_sum::AVG_FUNC: /* Place for sum & count */
if (group)
return new Field_string(sizeof(double)+sizeof(longlong),
maybe_null, item->name,table,&my_charset_bin);
0, item->name,table,&my_charset_bin);
else
return new Field_double(item_sum->max_length,maybe_null,
item->name, table, item_sum->decimals);
@ -4017,7 +4017,7 @@ Field *create_tmp_field(THD *thd, TABLE *table,Item *item, Item::Type type,
case Item_sum::STD_FUNC:
if (group)
return new Field_string(sizeof(double)*2+sizeof(longlong),
maybe_null, item->name,table,&my_charset_bin);
0, item->name,table,&my_charset_bin);
else
return new Field_double(item_sum->max_length, maybe_null,
item->name,table,item_sum->decimals);
@ -5621,11 +5621,11 @@ end_send(JOIN *join, JOIN_TAB *join_tab __attribute__((unused)),
TABLE *table=jt->table;
join->select_options ^= OPTION_FOUND_ROWS;
if (table->record_pointers ||
(table->io_cache && my_b_inited(table->io_cache)))
if (table->sort.record_pointers ||
(table->sort.io_cache && my_b_inited(table->sort.io_cache)))
{
/* Using filesort */
join->send_records= table->found_records;
join->send_records= table->sort.found_records;
}
else
{
@ -6460,8 +6460,8 @@ create_sort_index(THD *thd, JOIN_TAB *tab, ORDER *order,
if (!(sortorder=make_unireg_sortorder(order,&length)))
goto err; /* purecov: inspected */
/* It's not fatal if the following alloc fails */
table->io_cache=(IO_CACHE*) my_malloc(sizeof(IO_CACHE),
MYF(MY_WME | MY_ZEROFILL));
table->sort.io_cache=(IO_CACHE*) my_malloc(sizeof(IO_CACHE),
MYF(MY_WME | MY_ZEROFILL));
table->status=0; // May be wrong if quick_select
// If table has a range, move it to select
@ -6490,9 +6490,9 @@ create_sort_index(THD *thd, JOIN_TAB *tab, ORDER *order,
}
if (table->tmp_table)
table->file->info(HA_STATUS_VARIABLE); // Get record count
table->found_records=filesort(thd, table,sortorder, length,
select, filesort_limit, &examined_rows);
tab->records=table->found_records; // For SQL_CALC_ROWS
table->sort.found_records=filesort(thd, table,sortorder, length,
select, filesort_limit, &examined_rows);
tab->records=table->sort.found_records; // For SQL_CALC_ROWS
delete select; // filesort did select
tab->select=0;
tab->select_cond=0;
@ -6504,7 +6504,7 @@ create_sort_index(THD *thd, JOIN_TAB *tab, ORDER *order,
table->key_read=0;
table->file->extra(HA_EXTRA_NO_KEYREAD);
}
DBUG_RETURN(table->found_records == HA_POS_ERROR);
DBUG_RETURN(table->sort.found_records == HA_POS_ERROR);
err:
DBUG_RETURN(-1);
}

33
sql/sql_sort.h
View file

@ -19,6 +19,30 @@
#define MERGEBUFF 7
#define MERGEBUFF2 15
/*
The structure SORT_ADDON_FIELD describes a fixed layout
for field values appended to sorted values in records to be sorted
in the sort buffer.
Only fixed layout is supported now.
Null bit maps for the appended values is placed before the values
themselves. Offsets are from the last sorted field, that is from the
record referefence, which is still last component of sorted records.
It is preserved for backward compatiblility.
The structure is used tp store values of the additional fields
in the sort buffer. It is used also when these values are read
from a temporary file/buffer. As the reading procedures are beyond the
scope of the 'filesort' code the values have to be retrieved via
the callback function 'unpack_addon_fields'.
*/
typedef struct st_sort_addon_field { /* Sort addon packed field */
Field *field; /* Original field */
uint offset; /* Offset from the last sorted field */
uint null_offset; /* Offset to to null bit from the last sorted field */
uint length; /* Length in the sort buffer */
uint8 null_bit; /* Null bit mask for the field */
} SORT_ADDON_FIELD;
typedef struct st_buffpek { /* Struktur om sorteringsbuffrarna */
my_off_t file_pos; /* Where we are in the sort file */
uchar *base,*key; /* key pointers */
@ -27,15 +51,18 @@ typedef struct st_buffpek { /* Struktur om sorteringsbuffrarna */
ulong max_keys; /* Max keys in buffert */
} BUFFPEK;
typedef struct st_sort_param {
uint sort_length; /* Length of sort columns */
uint keys; /* Max keys / buffert */
uint rec_length; /* Length of sorted records */
uint sort_length; /* Length of sorted columns */
uint ref_length; /* Length of record ref. */
uint addon_length; /* Length of added packed fields */
uint res_length; /* Length of records in final sorted file/buffer */
uint keys; /* Max keys / buffer */
ha_rows max_rows,examined_rows;
TABLE *sort_form; /* For quicker make_sortkey */
SORT_FIELD *local_sortorder;
SORT_FIELD *end;
SORT_ADDON_FIELD *addon_field; /* Descriptors for companion fields */
uchar *unique_buff;
bool not_killable;
char* tmp_buffer;

10
sql/sql_table.cc
View file

@ -2342,8 +2342,8 @@ copy_data_between_tables(TABLE *from,TABLE *to,
if (order)
{
from->io_cache=(IO_CACHE*) my_malloc(sizeof(IO_CACHE),
MYF(MY_FAE | MY_ZEROFILL));
from->sort.io_cache=(IO_CACHE*) my_malloc(sizeof(IO_CACHE),
MYF(MY_FAE | MY_ZEROFILL));
bzero((char*) &tables,sizeof(tables));
tables.table = from;
tables.alias = tables.real_name= from->real_name;
@ -2355,9 +2355,9 @@ copy_data_between_tables(TABLE *from,TABLE *to,
setup_order(thd, thd->lex.select_lex.ref_pointer_array,
&tables, fields, all_fields, order) ||
!(sortorder=make_unireg_sortorder(order, &length)) ||
(from->found_records = filesort(thd, from, sortorder, length,
(SQL_SELECT *) 0, HA_POS_ERROR,
&examined_rows))
(from->sort.found_records = filesort(thd, from, sortorder, length,
(SQL_SELECT *) 0, HA_POS_ERROR,
&examined_rows))
== HA_POS_ERROR)
goto err;
};

4
sql/sql_update.cc
View file

@ -201,14 +201,14 @@ int mysql_update(THD *thd,
bzero((char*) &tables,sizeof(tables));
tables.table = table;
table->io_cache = (IO_CACHE *) my_malloc(sizeof(IO_CACHE),
table->sort.io_cache = (IO_CACHE *) my_malloc(sizeof(IO_CACHE),
MYF(MY_FAE | MY_ZEROFILL));
if (setup_ref_array(thd, &thd->lex.select_lex.ref_pointer_array,
order_num)||
setup_order(thd, thd->lex.select_lex.ref_pointer_array,
&tables, fields, all_fields, order) ||
!(sortorder=make_unireg_sortorder(order, &length)) ||
(table->found_records = filesort(thd, table, sortorder, length,
(table->sort.found_records = filesort(thd, table, sortorder, length,
(SQL_SELECT *) 0,
HA_POS_ERROR, &examined_rows))
== HA_POS_ERROR)

1
sql/structs.h
View file

@ -104,6 +104,7 @@ typedef struct st_read_record { /* Parameter to read_record */
uint index;
byte *ref_pos; /* pointer to form->refpos */
byte *record;
byte *rec_buf; /* to read field values after filesort */
byte *cache,*cache_pos,*cache_end,*read_positions;
IO_CACHE *io_cache;
bool print_error, ignore_not_found_rows;

15
sql/table.h
View file

@ -44,6 +44,17 @@ typedef struct st_grant_info
enum tmp_table_type {NO_TMP_TABLE=0, TMP_TABLE=1, TRANSACTIONAL_TMP_TABLE=2};
typedef struct st_filesort_info
{
IO_CACHE *io_cache; /* If sorted through filebyte */
byte *addon_buf; /* Pointer to a buffer if sorted with fields */
uint addon_length; /* Length of the buffer */
struct st_sort_addon_field *addon_field; /* Pointer to the fields info */
void (*unpack)(struct st_sort_addon_field *, byte *); /* To unpack back */
byte *record_pointers; /* If sorted in memory */
ha_rows found_records; /* How many records in sort */
} FILESORT_INFO;
/* Table cache entry struct */
class Field_timestamp;
@ -120,9 +131,7 @@ struct st_table {
table_map map; /* ID bit of table (1,2,4,8,16...) */
ulong version,flush_version;
uchar *null_flags;
IO_CACHE *io_cache; /* If sorted trough filebyte */
byte *record_pointers; /* If sorted in memory */
ha_rows found_records; /* How many records in sort */
FILESORT_INFO sort;
ORDER *group;
ha_rows quick_rows[MAX_KEY];
uint quick_key_parts[MAX_KEY];

10
sql/uniques.cc
View file

@ -95,12 +95,12 @@ bool Unique::flush()
bool Unique::get(TABLE *table)
{
SORTPARAM sort_param;
table->found_records=elements+tree.elements_in_tree;
table->sort.found_records=elements+tree.elements_in_tree;
if (my_b_tell(&file) == 0)
{
/* Whole tree is in memory; Don't use disk if you don't need to */
if ((record_pointers=table->record_pointers= (byte*)
if ((record_pointers=table->sort.record_pointers= (byte*)
my_malloc(tree.size_of_element * tree.elements_in_tree, MYF(0))))
{
(void) tree_walk(&tree, (tree_walk_action) unique_write_to_ptrs,
@ -112,7 +112,7 @@ bool Unique::get(TABLE *table)
if (flush())
return 1;
IO_CACHE *outfile=table->io_cache;
IO_CACHE *outfile=table->sort.io_cache;
BUFFPEK *file_ptr= (BUFFPEK*) file_ptrs.buffer;
uint maxbuffer= file_ptrs.elements - 1;
uchar *sort_buffer;
@ -120,8 +120,8 @@ bool Unique::get(TABLE *table)
bool error=1;
/* Open cached file if it isn't open */
outfile=table->io_cache=(IO_CACHE*) my_malloc(sizeof(IO_CACHE),
MYF(MY_ZEROFILL));
outfile=table->sort.io_cache=(IO_CACHE*) my_malloc(sizeof(IO_CACHE),
MYF(MY_ZEROFILL));
if (!outfile || ! my_b_inited(outfile) &&
open_cached_file(outfile,mysql_tmpdir,TEMP_PREFIX,READ_RECORD_BUFFER,