mirror of
https://github.com/MariaDB/server.git
synced 2025-01-29 02:05:57 +01:00
MDEV-5962: EITS: value "position" calculated incorrectly for CHAR(n) columns
- Dont substract unsigned numbers, use correct calculations. - (there is no testcase because effort is required to come up with it)
This commit is contained in:
parent
79a8a6130b
commit
aef0b4a897
1 changed files with 11 additions and 2 deletions
13
sql/field.cc
13
sql/field.cc
|
@ -1175,6 +1175,15 @@ inline ulonglong char_prefix_to_ulonglong(uchar *src)
|
|||
return uint8korr(src);
|
||||
}
|
||||
|
||||
/*
|
||||
Compute res = a - b, without losing precision and taking care that these are
|
||||
unsigned numbers.
|
||||
*/
|
||||
static inline double safe_substract(ulonglong a, ulonglong b)
|
||||
{
|
||||
return (a > b)? double(a - b) : -double(b - a);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
@brief
|
||||
|
@ -1227,10 +1236,10 @@ double Field::pos_in_interval_val_str(Field *min, Field *max, uint data_offset)
|
|||
minp= char_prefix_to_ulonglong(minp_prefix);
|
||||
maxp= char_prefix_to_ulonglong(maxp_prefix);
|
||||
double n, d;
|
||||
n= mp - minp;
|
||||
n= safe_substract(mp, minp);
|
||||
if (n < 0)
|
||||
return 0.0;
|
||||
d= maxp - minp;
|
||||
d= safe_substract(maxp, minp);
|
||||
if (d <= 0)
|
||||
return 1.0;
|
||||
return MY_MIN(n/d, 1.0);
|
||||
|
|
Loading…
Add table
Reference in a new issue