From 6769d1a0782f140dcd12c9ced6fda34ac0e41d85 Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Thu, 29 Oct 2020 01:40:31 +0100 Subject: [PATCH 01/18] MDEV-13467: Feature request: Support for ST_Distance_Sphere() - Cherry-pick 51e48b9f8981 - vscode gitignore - Thanks Robin Dupret for the review. Reviewed by:daniel@mariadb.org holyfoot@mariadb.com --- .gitignore | 10 + mysql-test/r/gis-precise.result | 111 +++++++++++ mysql-test/suite/innodb_gis/t/1.test | 13 -- .../suite/innodb_gis/t/bug16236208.test | 1 - .../innodb_gis/t/create_spatial_index.test | 1 - mysql-test/suite/innodb_gis/t/gis.test | 13 -- mysql-test/suite/innodb_gis/t/precise.test | 2 - mysql-test/suite/innodb_gis/t/rtree.test | 6 - mysql-test/t/gis-precise.test | 78 ++++++++ sql/item_create.cc | 36 +++- sql/item_geofunc.cc | 142 ++++++++++++- sql/item_geofunc.h | 14 ++ sql/spatial.cc | 188 ++++++++++++++++++ sql/spatial.h | 22 ++ 14 files changed, 598 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 8477515105b..00754884d3d 100644 --- a/.gitignore +++ b/.gitignore @@ -507,3 +507,13 @@ FakesAssemblies/ # QtCreator && CodeBlocks *.cbp +compile_commands.json +.clang-format +.kscope/ +.vimrc +.editorconfig +.kateconfig +*.kdev4 + +# Visual Studio Code workspace +.vscode/ diff --git a/mysql-test/r/gis-precise.result b/mysql-test/r/gis-precise.result index d5e1751d800..aaeb6e852a1 100644 --- a/mysql-test/r/gis-precise.result +++ b/mysql-test/r/gis-precise.result @@ -807,3 +807,114 @@ SRID(GEOMETRYFROMTEXT(' MULTIPOINT(8 4,5 0,7 8,6 9,3 4,7 3,5 5) ')))); ASTEXT(ST_BUFFER(POLYGONFROMTEXT(' POLYGON((9 9,5 2,4 5,9 9))'), SRID(GEOMETRYFROMTEXT(' MULTIPOINT(8 4,5 0,7 8,6 9,3 4,7 3,5 5) ')))) POLYGON((9 9,5 2,4 5,9 9)) +# +# MDEV-13467 Feature request: Support for ST_Distance_Sphere() +# +SELECT ST_DISTANCE_SPHERE(); +ERROR 42000: Incorrect parameter count in the call to native function 'ST_DISTANCE_SPHERE' +SELECT ST_DISTANCE_SPHERE(NULL); +ERROR 42000: Incorrect parameter count in the call to native function 'ST_DISTANCE_SPHERE' +SELECT ST_DISTANCE_SPHERE(NULL, NULL); +ST_DISTANCE_SPHERE(NULL, NULL) +NULL +SELECT ST_DISTANCE_SPHERE(NULL, NULL, 3); +ST_DISTANCE_SPHERE(NULL, NULL, 3) +NULL +SELECT ST_DISTANCE_SPHERE(NULL, 1, 3); +ST_DISTANCE_SPHERE(NULL, 1, 3) +NULL +SELECT ST_DISTANCE_SPHERE(1, NULL, 3); +ST_DISTANCE_SPHERE(1, NULL, 3) +NULL +SELECT ST_DISTANCE_SPHERE(1, 1); +ERROR 22023: Invalid GIS data provided to function ST_Distance_Sphere. +SELECT ST_DISTANCE_SPHERE(1, 1, 3); +ERROR 22023: Invalid GIS data provided to function ST_Distance_Sphere. +SELECT ST_DISTANCE_SPHERE(1, 1, NULL); +ST_DISTANCE_SPHERE(1, 1, NULL) +NULL +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(1 0)'), ST_GEOMFROMTEXT('LINESTRING(0 0, 1 1)')); +ERROR HY000: Internal error: st_distance_sphere +# Test Points and radius +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('POINT(1 1)')); +ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('POINT(1 1)')) +157249.0357231545 +SELECT TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(-1 -1)'), ST_GEOMFROMTEXT('POINT(-2 -2)')), 10); +TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(-1 -1)'), ST_GEOMFROMTEXT('POINT(-2 -2)')), 10) +157225.0865419108 +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('POINT(1 1)'), 1); +ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('POINT(1 1)'), 1) +0.024682056391766436 +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('POINT(1 1)'), 0); +ERROR HY000: Internal error: Radius must be greater than zero. +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('POINT(1 1)'), -1); +ERROR HY000: Internal error: Radius must be greater than zero. +# Test longitude/lattitude +SELECT TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 1)'), ST_GEOMFROMTEXT('POINT(1 2)')), 10); +TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 1)'), ST_GEOMFROMTEXT('POINT(1 2)')), 10) +157225.0865419108 +SELECT TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 1)'), ST_GEOMFROMTEXT('POINT(2 1)')), 10); +TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 1)'), ST_GEOMFROMTEXT('POINT(2 1)')), 10) +222355.4901806686 +SELECT TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(1 0)'), ST_GEOMFROMTEXT('POINT(1 2)')), 10); +TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(1 0)'), ST_GEOMFROMTEXT('POINT(1 2)')), 10) +222389.3645969269 +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(1 0)'), ST_GEOMFROMTEXT('POINT(2 1)')); +ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(1 0)'), ST_GEOMFROMTEXT('POINT(2 1)')) +157249.0357231545 +# Test Points - Multipoints +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(1 1)')); +ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(1 1)')) +157249.0357231545 +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(1 1)'), ST_GEOMFROMTEXT('POINT(0 0)')); +ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(1 1)'), ST_GEOMFROMTEXT('POINT(0 0)')) +157249.0357231545 +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(1 1,2 2)')); +ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(1 1,2 2)')) +157249.0357231545 +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(2 2,1 1)')); +ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(2 2,1 1)')) +157249.0357231545 +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(1 1,2 2)'), 1); +ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(1 1,2 2)'), 1) +0.024682056391766436 +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(2 2,1 1)'), 1); +ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(2 2,1 1)'), 1) +0.024682056391766436 +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(2 2, 1 1, 3 4)'), 1); +ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(2 2, 1 1, 3 4)'), 1) +0.024682056391766436 +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(2 2, 1 1,5 6)'), 1); +ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(2 2, 1 1,5 6)'), 1) +0.024682056391766436 +# Test Multipoints - Multipoints +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(3 4,8 9 )'), ST_GEOMFROMTEXT('MULTIPOINT(3 4,8 9 )')); +ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(3 4,8 9 )'), ST_GEOMFROMTEXT('MULTIPOINT(3 4,8 9 )')) +0 +SELECT TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(1 2,1 1 )'), ST_GEOMFROMTEXT('MULTIPOINT(3 4,8 9 )')), 10); +TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(1 2,1 1 )'), ST_GEOMFROMTEXT('MULTIPOINT(3 4,8 9 )')), 10) +314282.5644496733 +SELECT TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(1 2,1 1 )'), ST_GEOMFROMTEXT('MULTIPOINT(8 9,3 4 )')), 10); +TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(1 2,1 1 )'), ST_GEOMFROMTEXT('MULTIPOINT(8 9,3 4 )')), 10) +314282.5644496733 +SELECT TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(1 2,1 1 )'), ST_GEOMFROMTEXT('MULTIPOINT(8 9,3 4 )'),1), 17); +TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(1 2,1 1 )'), ST_GEOMFROMTEXT('MULTIPOINT(8 9,3 4 )'),1), 17) +0.04933028646581131 +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(1 2,1 1 )'), ST_GEOMFROMTEXT('MULTIPOINT(8 9,3 4 )'),0); +ERROR HY000: Internal error: Radius must be greater than zero. +set @pt1 = ST_GeomFromText('POINT(190 -30)'); +set @pt2 = ST_GeomFromText('POINT(-30 50)'); +SELECT ST_Distance_Sphere(@pt1, @pt2); +ERROR HY000: Out of range error: Longitude should be [-180,180] in function ST_Distance_Sphere. +set @pt1 = ST_GeomFromText('POINT(135 -30)'); +set @pt2 = ST_GeomFromText('POINT(-30 91)'); +SELECT ST_Distance_Sphere(@pt1, @pt2); +ERROR HY000: Out of range error: Latitude should be [-90,90] in function ST_Distance_Sphere. +set @zenica = ST_GeomFromText('POINT(17.907743 44.203438)'); +set @sarajevo = ST_GeomFromText('POINT(18.413076 43.856258)'); +SELECT TRUNCATE(ST_Distance_Sphere(@zenica, @sarajevo), 10); +TRUNCATE(ST_Distance_Sphere(@zenica, @sarajevo), 10) +55878.5933759170 +SELECT TRUNCATE(ST_Distance_Sphere(@sarajevo, @zenica), 10); +TRUNCATE(ST_Distance_Sphere(@sarajevo, @zenica), 10) +55878.5933759170 diff --git a/mysql-test/suite/innodb_gis/t/1.test b/mysql-test/suite/innodb_gis/t/1.test index 256cd1ab0ad..0e5fba6fc34 100644 --- a/mysql-test/suite/innodb_gis/t/1.test +++ b/mysql-test/suite/innodb_gis/t/1.test @@ -393,7 +393,6 @@ insert into t1 values (1); insert into t1 values (1.11); --error 1416 insert into t1 values ("qwerty"); -# --error ER_GIS_INVALID_DATA --error ER_BAD_NULL_ERROR insert into t1 values (ST_pointfromtext('point(1,1)')); @@ -437,7 +436,6 @@ select ST_y(b) IS NULL from t1; -# --error ER_GIS_INVALID_DATA select MBRwithin(b, b) IS NULL, MBRcontains(b, b) IS NULL, MBRoverlaps(b, b) IS NULL, MBRequals(b, b) IS NULL, MBRdisjoint(b, b) IS NULL, ST_touches(b, b) IS NULL, @@ -466,7 +464,6 @@ DROP TABLE t1; # CREATE TABLE `t1` ( `col9` set('a'), `col89` date); INSERT IGNORE INTO `t1` VALUES ('','0000-00-00'); -# --error ER_GIS_INVALID_DATA select ST_geomfromtext(col9,col89) as a from t1; DROP TABLE t1; @@ -623,17 +620,11 @@ SELECT GROUP_CONCAT(a2.name ORDER BY a2.name) AS MBRwithin FROM t1 a1 JOIN # MBROverlaps needs a few more tests, with point and line dimensions -# --error ER_GIS_INVALID_DATA SET @vert1 = ST_GeomFromText('POLYGON ((0 -2, 0 2, 0 -2))'); -# --error ER_GIS_INVALID_DATA SET @horiz1 = ST_GeomFromText('POLYGON ((-2 0, 2 0, -2 0))'); -# --error ER_GIS_INVALID_DATA SET @horiz2 = ST_GeomFromText('POLYGON ((-1 0, 3 0, -1 0))'); -# --error ER_GIS_INVALID_DATA SET @horiz3 = ST_GeomFromText('POLYGON ((2 0, 3 0, 2 0))'); -# --error ER_GIS_INVALID_DATA SET @point1 = ST_GeomFromText('POLYGON ((0 0))'); -# --error ER_GIS_INVALID_DATA SET @point2 = ST_GeomFromText('POLYGON ((-2 0))'); SELECT GROUP_CONCAT(a1.name ORDER BY a1.name) AS MBRoverlaps FROM t1 a1 WHERE MBROverlaps(a1.square, @vert1) GROUP BY a1.name; @@ -772,10 +763,8 @@ SELECT 1 FROM (SELECT GREATEST(1,GEOMETRYCOLLECTION('00000','00000')) b FROM DUA --echo # BUG#51875: crash when loading data into geometry function ST_polyfromwkb --echo # SET @a=0x00000000030000000100000000000000000000000000144000000000000014400000000000001840000000000000184000000000000014400000000000001440; -# --error ER_GIS_INVALID_DATA SET @a=ST_POLYFROMWKB(@a); SET @a=0x00000000030000000000000000000000000000000000144000000000000014400000000000001840000000000000184000000000000014400000000000001440; -# --error ER_GIS_INVALID_DATA SET @a=ST_POLYFROMWKB(@a); @@ -899,7 +888,6 @@ DROP TABLE g1; CREATE TABLE g1(a TEXT NOT NULL, KEY(a(255))); INSERT INTO g1 VALUES ('a'),('a'); -# --error ER_GIS_INVALID_DATA SELECT 1 FROM g1 WHERE a >= ANY (SELECT 1 FROM g1 WHERE a = ST_geomfromtext('') OR a) ; @@ -1447,5 +1435,4 @@ DROP DATABASE gis_ogs; --echo # Bug#13362660 ASSERTION `FIELD_POS < FIELD_COUNT' FAILED. IN PROTOCOL_TEXT::STORE --echo # -# --error ER_GIS_INVALID_DATA SELECT ST_Union('', ''), md5(1); diff --git a/mysql-test/suite/innodb_gis/t/bug16236208.test b/mysql-test/suite/innodb_gis/t/bug16236208.test index b55ab1d0fd3..3a1fbefc52c 100644 --- a/mysql-test/suite/innodb_gis/t/bug16236208.test +++ b/mysql-test/suite/innodb_gis/t/bug16236208.test @@ -47,7 +47,6 @@ ST_GeomFromText('GEOMETRYCOLLECTION(POINT(0 0), LINESTRING(0 0,10 10))')); CREATE INDEX linestring_index ON linestring(linestring_nokey(5)); ALTER TABLE linestring ADD KEY (linestring_key(5)); -# --error ER_GIS_INVALID_DATA SELECT ST_AsText(linestring_nokey) FROM linestring FORCE KEY ( linestring_key ) WHERE ST_CONTAINS( ST_GeomFromText('POLYGON( ( 3923 2815 , 4246 2122 , 4028 2971 , 4017 3019 , 3923 2815 ) )') , linestring_key ) AND diff --git a/mysql-test/suite/innodb_gis/t/create_spatial_index.test b/mysql-test/suite/innodb_gis/t/create_spatial_index.test index 6a15b0f2d74..1c0a7e7aaa7 100644 --- a/mysql-test/suite/innodb_gis/t/create_spatial_index.test +++ b/mysql-test/suite/innodb_gis/t/create_spatial_index.test @@ -1141,7 +1141,6 @@ insert into `t1` values( linestring(point(1,1),point(1,1)) ); -# --error ER_GIS_INVALID_DATA --error ER_BAD_NULL_ERROR insert into `t1` values ( diff --git a/mysql-test/suite/innodb_gis/t/gis.test b/mysql-test/suite/innodb_gis/t/gis.test index 489934e957e..ba1b3b12ec2 100644 --- a/mysql-test/suite/innodb_gis/t/gis.test +++ b/mysql-test/suite/innodb_gis/t/gis.test @@ -388,7 +388,6 @@ insert into t1 values (1.11); --error 1416 insert into t1 values ("qwerty"); --error 1048 -# --error ER_GIS_INVALID_DATA insert into t1 values (ST_pointfromtext('point(1,1)')); drop table t1; @@ -431,7 +430,6 @@ select ST_y(b) IS NULL from t1; -# --error ER_GIS_INVALID_DATA select MBRwithin(b, b) IS NULL, MBRcontains(b, b) IS NULL, MBRoverlaps(b, b) IS NULL, MBRequals(b, b) IS NULL, MBRdisjoint(b, b) IS NULL, ST_touches(b, b) IS NULL, @@ -460,7 +458,6 @@ DROP TABLE t1; # CREATE TABLE `t1` ( `col9` set('a'), `col89` date); INSERT IGNORE INTO `t1` VALUES ('','0000-00-00'); -# --error ER_GIS_INVALID_DATA select ST_geomfromtext(col9,col89) as a from t1; DROP TABLE t1; @@ -617,17 +614,11 @@ SELECT GROUP_CONCAT(a2.name ORDER BY a2.name) AS MBRwithin FROM t1 a1 JOIN # MBROverlaps needs a few more tests, with point and line dimensions -# --error ER_GIS_INVALID_DATA SET @vert1 = ST_GeomFromText('POLYGON ((0 -2, 0 2, 0 -2))'); -# --error ER_GIS_INVALID_DATA SET @horiz1 = ST_GeomFromText('POLYGON ((-2 0, 2 0, -2 0))'); -# --error ER_GIS_INVALID_DATA SET @horiz2 = ST_GeomFromText('POLYGON ((-1 0, 3 0, -1 0))'); -# --error ER_GIS_INVALID_DATA SET @horiz3 = ST_GeomFromText('POLYGON ((2 0, 3 0, 2 0))'); -# --error ER_GIS_INVALID_DATA SET @point1 = ST_GeomFromText('POLYGON ((0 0))'); -# --error ER_GIS_INVALID_DATA SET @point2 = ST_GeomFromText('POLYGON ((-2 0))'); SELECT GROUP_CONCAT(a1.name ORDER BY a1.name) AS MBRoverlaps FROM t1 a1 WHERE MBROverlaps(a1.square, @vert1) GROUP BY a1.name; @@ -766,10 +757,8 @@ SELECT 1 FROM (SELECT GREATEST(1,GEOMETRYCOLLECTION('00000','00000')) b FROM DUA --echo # BUG#51875: crash when loading data into geometry function ST_polyfromwkb --echo # SET @a=0x00000000030000000100000000000000000000000000144000000000000014400000000000001840000000000000184000000000000014400000000000001440; -# --error ER_GIS_INVALID_DATA SET @a=ST_POLYFROMWKB(@a); SET @a=0x00000000030000000000000000000000000000000000144000000000000014400000000000001840000000000000184000000000000014400000000000001440; -# --error ER_GIS_INVALID_DATA SET @a=ST_POLYFROMWKB(@a); @@ -901,7 +890,6 @@ DROP TABLE g1; CREATE TABLE g1(a TEXT NOT NULL, KEY(a(255))); INSERT INTO g1 VALUES ('a'),('a'); -# --error ER_GIS_INVALID_DATA SELECT 1 FROM g1 WHERE a >= ANY (SELECT 1 FROM g1 WHERE a = ST_geomfromtext('') OR a) ; @@ -1440,5 +1428,4 @@ DROP DATABASE gis_ogs; --echo # Bug#13362660 ASSERTION `FIELD_POS < FIELD_COUNT' FAILED. IN PROTOCOL_TEXT::STORE --echo # -# --error ER_GIS_INVALID_DATA SELECT ST_Union('', ''), md5(1); diff --git a/mysql-test/suite/innodb_gis/t/precise.test b/mysql-test/suite/innodb_gis/t/precise.test index 9174a100003..3936610530f 100644 --- a/mysql-test/suite/innodb_gis/t/precise.test +++ b/mysql-test/suite/innodb_gis/t/precise.test @@ -122,7 +122,6 @@ SELECT ST_Equals(ST_PointFromText('POINT (12 13)'),ST_PointFromText('POINT (12 1 --echo # BUG#11759650/51979: UNION/INTERSECTION OF POLYGONS CRASHES MYSQL --echo # -# --error ER_GIS_INVALID_DATA SELECT ST_ASTEXT(ST_UNION(ST_GEOMFROMTEXT('POLYGON((525000 183300,525400 183300,525400 18370, 525000 183700,525000 183300))'), ST_geomfromtext('POLYGON((525298.67 183511.53,525296.57 @@ -140,7 +139,6 @@ ST_geomfromtext('POLYGON((525298.67 183511.53,525296.57 183491.55))'))) st_u; SET @a=0x0000000001030000000200000005000000000000000000000000000000000000000000000000002440000000000000000000000000000024400000000000002440000000000000000000000000000024400000000000000000000000000000000000000000000000000000F03F000000000000F03F0000000000000040000000000000F03F00000000000000400000000000000040000000000000F03F0000000000000040000000000000F03F000000000000F03F; -# --error ER_GIS_INVALID_DATA SELECT ST_ASTEXT(ST_TOUCHES(@a, ST_GEOMFROMTEXT('point(0 0)'))) t; diff --git a/mysql-test/suite/innodb_gis/t/rtree.test b/mysql-test/suite/innodb_gis/t/rtree.test index 58d81576b3e..98931e70e62 100644 --- a/mysql-test/suite/innodb_gis/t/rtree.test +++ b/mysql-test/suite/innodb_gis/t/rtree.test @@ -78,17 +78,11 @@ SELECT name, ST_AsText(square) from t1 where MBRWithin(@p, square); # MBROverlaps needs a few more tests, with point and line dimensions -# --error ER_GIS_INVALID_DATA SET @vert1 = ST_GeomFromText('POLYGON ((0 -2, 0 2, 0 -2))'); -# --error ER_GIS_INVALID_DATA SET @horiz1 = ST_GeomFromText('POLYGON ((-2 0, 2 0, -2 0))'); -# --error ER_GIS_INVALID_DATA SET @horiz2 = ST_GeomFromText('POLYGON ((-1 0, 3 0, -1 0))'); -# --error ER_GIS_INVALID_DATA SET @horiz3 = ST_GeomFromText('POLYGON ((2 0, 3 0, 2 0))'); -# --error ER_GIS_INVALID_DATA SET @point1 = ST_GeomFromText('POLYGON ((0 0))'); -# --error ER_GIS_INVALID_DATA SET @point2 = ST_GeomFromText('POLYGON ((-2 0))'); SELECT GROUP_CONCAT(a1.name ORDER BY a1.name) AS MBRoverlaps FROM t1 a1 WHERE MBROverlaps(a1.square, @vert1) GROUP BY a1.name; diff --git a/mysql-test/t/gis-precise.test b/mysql-test/t/gis-precise.test index ede535596da..b75318e034b 100644 --- a/mysql-test/t/gis-precise.test +++ b/mysql-test/t/gis-precise.test @@ -393,3 +393,81 @@ with cte1 as( select (st_symdifference(point(1,1),point(1,1))) as a1 ), cte2 a --source include/gis_debug.inc + +--echo # +--echo # MDEV-13467 Feature request: Support for ST_Distance_Sphere() +--echo # + +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT ST_DISTANCE_SPHERE(); +--error ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT +SELECT ST_DISTANCE_SPHERE(NULL); +SELECT ST_DISTANCE_SPHERE(NULL, NULL); +# NULL args and optional radius will return NULL +SELECT ST_DISTANCE_SPHERE(NULL, NULL, 3); +# At least 1 NULL arg and optional radius will return NULL +SELECT ST_DISTANCE_SPHERE(NULL, 1, 3); +# At least 1 NULL arg and optional radius will return NULL +SELECT ST_DISTANCE_SPHERE(1, NULL, 3); +# Return ER_GIS_INVALID_DATA for invalid geometry +--error ER_GIS_INVALID_DATA +SELECT ST_DISTANCE_SPHERE(1, 1); +--error ER_GIS_INVALID_DATA +SELECT ST_DISTANCE_SPHERE(1, 1, 3); +# Return NULL if radius is NULL +SELECT ST_DISTANCE_SPHERE(1, 1, NULL); +# Wrong geometry +--error ER_INTERNAL_ERROR +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(1 0)'), ST_GEOMFROMTEXT('LINESTRING(0 0, 1 1)')); + +--echo # Test Points and radius +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('POINT(1 1)')); +# make bb x86 happy +SELECT TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(-1 -1)'), ST_GEOMFROMTEXT('POINT(-2 -2)')), 10); +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('POINT(1 1)'), 1); +--error ER_INTERNAL_ERROR +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('POINT(1 1)'), 0); +--error ER_INTERNAL_ERROR +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('POINT(1 1)'), -1); +--echo # Test longitude/lattitude +# make bb x86 happy +SELECT TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 1)'), ST_GEOMFROMTEXT('POINT(1 2)')), 10); +SELECT TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 1)'), ST_GEOMFROMTEXT('POINT(2 1)')), 10); +SELECT TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(1 0)'), ST_GEOMFROMTEXT('POINT(1 2)')), 10); +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(1 0)'), ST_GEOMFROMTEXT('POINT(2 1)')); +--echo # Test Points - Multipoints +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(1 1)')); +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(1 1)'), ST_GEOMFROMTEXT('POINT(0 0)')); +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(1 1,2 2)')); +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(2 2,1 1)')); +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(1 1,2 2)'), 1); +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(2 2,1 1)'), 1); +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(2 2, 1 1, 3 4)'), 1); +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('POINT(0 0)'), ST_GEOMFROMTEXT('MULTIPOINT(2 2, 1 1,5 6)'), 1); +--echo # Test Multipoints - Multipoints +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(3 4,8 9 )'), ST_GEOMFROMTEXT('MULTIPOINT(3 4,8 9 )')); +# make bb x86 happy +SELECT TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(1 2,1 1 )'), ST_GEOMFROMTEXT('MULTIPOINT(3 4,8 9 )')), 10); +SELECT TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(1 2,1 1 )'), ST_GEOMFROMTEXT('MULTIPOINT(8 9,3 4 )')), 10); +# make bb x86 happy +SELECT TRUNCATE(ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(1 2,1 1 )'), ST_GEOMFROMTEXT('MULTIPOINT(8 9,3 4 )'),1), 17); +--error ER_INTERNAL_ERROR +SELECT ST_DISTANCE_SPHERE(ST_GEOMFROMTEXT('MULTIPOINT(1 2,1 1 )'), ST_GEOMFROMTEXT('MULTIPOINT(8 9,3 4 )'),0); + +# Longitude out of range [-180,180] +set @pt1 = ST_GeomFromText('POINT(190 -30)'); +set @pt2 = ST_GeomFromText('POINT(-30 50)'); +--error ER_STD_OUT_OF_RANGE_ERROR +SELECT ST_Distance_Sphere(@pt1, @pt2); + +# Latitude out of range [-90, 90] +set @pt1 = ST_GeomFromText('POINT(135 -30)'); +set @pt2 = ST_GeomFromText('POINT(-30 91)'); +--error ER_STD_OUT_OF_RANGE_ERROR +SELECT ST_Distance_Sphere(@pt1, @pt2); + +# POINT in form (longitude[-180, 180] latitude[-90, 90]) +set @zenica = ST_GeomFromText('POINT(17.907743 44.203438)'); +set @sarajevo = ST_GeomFromText('POINT(18.413076 43.856258)'); +SELECT TRUNCATE(ST_Distance_Sphere(@zenica, @sarajevo), 10); +SELECT TRUNCATE(ST_Distance_Sphere(@sarajevo, @zenica), 10); diff --git a/sql/item_create.cc b/sql/item_create.cc index 1cf5a06a3a4..4bfb2615fbe 100644 --- a/sql/item_create.cc +++ b/sql/item_create.cc @@ -202,7 +202,6 @@ public: @return An item representing the function call */ virtual Item *create_2_arg(THD *thd, Item *arg1, Item *arg2) = 0; - protected: /** Constructor. */ Create_func_arg2() {} @@ -229,7 +228,6 @@ public: @return An item representing the function call */ virtual Item *create_3_arg(THD *thd, Item *arg1, Item *arg2, Item *arg3) = 0; - protected: /** Constructor. */ Create_func_arg3() {} @@ -975,6 +973,19 @@ class Create_func_distance : public Create_func_arg2 Create_func_distance() {} virtual ~Create_func_distance() {} }; + + +class Create_func_distance_sphere: public Create_native_func +{ + public: + virtual Item *create_native(THD *thd, LEX_STRING name, List *item_list); + static Create_func_distance_sphere s_singleton; + + protected: + Create_func_distance_sphere() {} + virtual ~Create_func_distance_sphere() {} +}; + #endif @@ -4761,6 +4772,26 @@ Create_func_glength::create_1_arg(THD *thd, Item *arg1) { return new (thd->mem_root) Item_func_glength(thd, arg1); } + + +Create_func_distance_sphere Create_func_distance_sphere::s_singleton; + +Item* +Create_func_distance_sphere::create_native(THD *thd, LEX_STRING name, + List *item_list) +{ + int arg_count= 0; + + if (item_list != NULL) + arg_count= item_list->elements; + + if (arg_count < 2) + { + my_error(ER_WRONG_PARAMCOUNT_TO_NATIVE_FCT, MYF(0), name.str); + return NULL; + } + return new (thd->mem_root) Item_func_sphere_distance(thd, *item_list); +} #endif @@ -7051,6 +7082,7 @@ static Native_func_registry func_array[] = { { C_STRING_WITH_LEN("ST_WITHIN") }, GEOM_BUILDER(Create_func_within)}, { { C_STRING_WITH_LEN("ST_X") }, GEOM_BUILDER(Create_func_x)}, { { C_STRING_WITH_LEN("ST_Y") }, GEOM_BUILDER(Create_func_y)}, + { { C_STRING_WITH_LEN("ST_DISTANCE_SPHERE") }, GEOM_BUILDER(Create_func_distance_sphere)}, { { C_STRING_WITH_LEN("SUBSTRING_INDEX") }, BUILDER(Create_func_substr_index)}, { { C_STRING_WITH_LEN("SUBTIME") }, BUILDER(Create_func_subtime)}, { { C_STRING_WITH_LEN("TAN") }, BUILDER(Create_func_tan)}, diff --git a/sql/item_geofunc.cc b/sql/item_geofunc.cc index 1c3fafc0582..2b0f67f4ddc 100644 --- a/sql/item_geofunc.cc +++ b/sql/item_geofunc.cc @@ -2537,11 +2537,151 @@ mem_error: } +double Item_func_sphere_distance::val_real() +{ + /* To test null_value of item, first get well-known bytes as a backups */ + String bak1, bak2; + String *arg1= args[0]->val_str(&bak1); + String *arg2= args[1]->val_str(&bak2); + double distance= 0.0; + double sphere_radius= 6370986.0; // Default radius equals Earth radius + + null_value= (args[0]->null_value || args[1]->null_value); + if (null_value) + { + goto handle_errors; + } + + if (arg_count == 3) + { + sphere_radius= args[2]->val_real(); + // Radius cannot be Null + if (args[2]->null_value) + { + null_value= true; + goto handle_errors; + } + if (sphere_radius <= 0) + { + my_error(ER_INTERNAL_ERROR, MYF(0), "Radius must be greater than zero."); + return 1; + } + } + Geometry_buffer buffer1, buffer2; + Geometry *g1, *g2; + if (!(g1= Geometry::construct(&buffer1, arg1->ptr(), arg1->length())) || + !(g2= Geometry::construct(&buffer2, arg2->ptr(), arg2->length()))) + { + my_error(ER_GIS_INVALID_DATA, MYF(0), "ST_Distance_Sphere"); + goto handle_errors; + } +// Method allowed for points and multipoints + if (!(g1->get_class_info()->m_type_id == Geometry::wkb_point || + g1->get_class_info()->m_type_id == Geometry::wkb_multipoint) || + !(g2->get_class_info()->m_type_id == Geometry::wkb_point || + g2->get_class_info()->m_type_id == Geometry::wkb_multipoint)) + { + // Generate error message in case different geometry is used? + my_error(ER_INTERNAL_ERROR, MYF(0), func_name()); + return 0; + } + distance= spherical_distance_points(g1, g2, sphere_radius); + if (distance < 0) + { + my_error(ER_INTERNAL_ERROR, MYF(0), "Returned distance cannot be negative."); + return 1; + } + return distance; + + handle_errors: + return 0; +} + + +double Item_func_sphere_distance::spherical_distance_points(Geometry *g1, + Geometry *g2, + const double r) +{ + double res= 0.0; + // Length for the single point (25 Bytes) + uint32 len= SRID_SIZE + POINT_DATA_SIZE + WKB_HEADER_SIZE; + int error= 0; + + switch (g2->get_class_info()->m_type_id) + { + case Geometry::wkb_point: + // Optimization for point-point case + if (g1->get_class_info()->m_type_id == Geometry::wkb_point) + { + res= static_cast(g2)->calculate_haversine(g1, r, &error); + } + else + { + // Optimization for single point in Multipoint + if (g1->get_data_size() == len) + { + res= static_cast(g2)->calculate_haversine(g1, r, &error); + } + else + { + // There are multipoints in g1 + // g1 is MultiPoint and calculate MP.sphericaldistance from g2 Point + if (g1->get_data_size() != GET_SIZE_ERROR) + static_cast(g2)->spherical_distance_multipoints( + (Gis_multi_point *)g1, r, &res, &error); + } + } + break; + + case Geometry::wkb_multipoint: + // Optimization for point-point case + if (g1->get_class_info()->m_type_id == Geometry::wkb_point) + { + // Optimization for single point in Multipoint g2 + if (g2->get_data_size() == len) + { + res= static_cast(g1)->calculate_haversine(g2, r, &error); + } + else + { + if (g2->get_data_size() != GET_SIZE_ERROR) + // g1 is a point (casted to multi_point) and g2 multipoint + static_cast(g1)->spherical_distance_multipoints( + (Gis_multi_point *)g2, r, &res, &error); + } + } + else + { + // Multipoints in g1 and g2 - no optimization + static_cast(g1)->spherical_distance_multipoints( + (Gis_multi_point *)g2, r, &res, &error); + } + break; + + default: + DBUG_ASSERT(0); + break; + } + + if (res < 0) + goto handle_error; + + handle_error: + if (error > 0) + my_error(ER_STD_OUT_OF_RANGE_ERROR, MYF(0), + "Longitude should be [-180,180]", "ST_Distance_Sphere"); + else if(error < 0) + my_error(ER_STD_OUT_OF_RANGE_ERROR, MYF(0), + "Latitude should be [-90,90]", "ST_Distance_Sphere"); + return res; +} + + String *Item_func_pointonsurface::val_str(String *str) { Gcalc_operation_transporter trn(&func, &collector); - DBUG_ENTER("Item_func_pointonsurface::val_real"); + DBUG_ENTER("Item_func_pointonsurface::val_str"); DBUG_ASSERT(fixed == 1); String *res= args[0]->val_str(&tmp_value); Geometry_buffer buffer; diff --git a/sql/item_geofunc.h b/sql/item_geofunc.h index a858a9a2d57..efff940ec4b 100644 --- a/sql/item_geofunc.h +++ b/sql/item_geofunc.h @@ -714,6 +714,20 @@ public: }; +class Item_func_sphere_distance: public Item_real_func +{ + double spherical_distance_points(Geometry *g1, Geometry *g2, + const double sphere_r); +public: + Item_func_sphere_distance(THD *thd, List &list): + Item_real_func(thd, list) {} + double val_real(); + const char *func_name() const { return "st_distance_sphere"; } + Item *get_copy(THD *thd, MEM_ROOT *mem_root) + { return get_item_copy(thd, mem_root, this); } +}; + + class Item_func_pointonsurface: public Item_geometry_func { String tmp_value; diff --git a/sql/spatial.cc b/sql/spatial.cc index 840f8bd809c..711efa91394 100644 --- a/sql/spatial.cc +++ b/sql/spatial.cc @@ -1032,6 +1032,119 @@ const Geometry::Class_info *Gis_point::get_class_info() const } +/** + Function to calculate haversine. + Taking as arguments Point and Multipoint geometries. + Multipoint geometry has to be single point only. + It is up to caller to ensure valid input. + + @param g pointer to the Geometry + @param r sphere radius + @param error pointer describing the error in case of the boundary conditions + + @return distance in case without error, it is caclulcated distance (non-negative), + in case error exist, negative value. +*/ +double Gis_point::calculate_haversine(const Geometry *g, + const double sphere_radius, + int *error) +{ + DBUG_ASSERT(sphere_radius > 0); + double x1r, x2r, y1r, y2r, dlong, dlat, res; + + // This check is done only for optimization purposes where we know it will + // be one and only one point in Multipoint + if (g->get_class_info()->m_type_id == Geometry::wkb_multipoint) + { + const char point_size= 4 + WKB_HEADER_SIZE + POINT_DATA_SIZE+1; //1 for the type + char point_temp[point_size]; + memset(point_temp+4, Geometry::wkb_point, 1); + memcpy(point_temp+5, static_cast(g)->get_data_ptr()+5, 4); + memcpy(point_temp+4+WKB_HEADER_SIZE, g->get_data_ptr()+4+WKB_HEADER_SIZE, + POINT_DATA_SIZE); + point_temp[point_size-1]= '\0'; + Geometry_buffer gbuff; + Geometry *gg= Geometry::construct(&gbuff, point_temp, point_size-1); + DBUG_ASSERT(gg); + if (static_cast(gg)->get_xy_radian(&x2r, &y2r)) + DBUG_ASSERT(0); + } + else + { + if (static_cast(g)->get_xy_radian(&x2r, &y2r)) + DBUG_ASSERT(0); + } + if (this->get_xy_radian(&x1r, &y1r)) + DBUG_ASSERT(0); + // Check boundary conditions: longitude[-180,180] + if (!((x2r >= -M_PI && x2r <= M_PI) && (x1r >= -M_PI && x1r <= M_PI))) + { + *error=1; + return -1; + } + // Check boundary conditions: lattitude[-90,90] + if (!((y2r >= -M_PI/2 && y2r <= M_PI/2) && (y1r >= -M_PI/2 && y1r <= M_PI/2))) + { + *error=-1; + return -1; + } + dlat= sin((y2r - y1r)/2)*sin((y2r - y1r)/2); + dlong= sin((x2r - x1r)/2)*sin((x2r - x1r)/2); + res= 2*sphere_radius*asin((sqrt(dlat + cos(y1r)*cos(y2r)*dlong))); + return res; +} + + +/** + Function that calculate spherical distance of Point from Multipoint geometries. + In case there is single point in Multipoint geometries calculate_haversine() + can handle such case. Otherwise, new geometry (Point) has to be constructed. + + @param g pointer to the Geometry + @param r sphere radius + @param result pointer to the result + @param err pointer to the error obtained from calculate_haversin() + + @return state + @retval TRUE failed + @retval FALSE success +*/ +int Gis_point::spherical_distance_multipoints(Geometry *g, const double r, + double *result, int *err) +{ + uint32 num_of_points2; + // To find the minimum radius it cannot be greater than Earth radius + double res= 6370986.0; + double temp_res= 0.0; + const uint32 len= 4 + WKB_HEADER_SIZE + POINT_DATA_SIZE + 1; + char s[len]; + g->num_geometries(&num_of_points2); + DBUG_ASSERT(num_of_points2 >= 1); + if (num_of_points2 == 1) + { + *result= this->calculate_haversine(g, r, err); + return 0; + } + for (uint32 i=1; i <= num_of_points2; i++) + { + Geometry_buffer buff_temp; + Geometry *temp; + + // First 4 bytes are handled already, make sure to create a Point + memset(s + 4, Geometry::wkb_point, 1); + memcpy(s + 5, g->get_data_ptr() + 5, 4); + memcpy(s + 4 + WKB_HEADER_SIZE, g->get_data_ptr() + 4 + WKB_HEADER_SIZE*i +\ + POINT_DATA_SIZE*(i-1), POINT_DATA_SIZE); + s[len-1]= '\0'; + temp= Geometry::construct(&buff_temp, s, len); + DBUG_ASSERT(temp); + temp_res= this->calculate_haversine(temp, r, err); + if (res > temp_res) + res= temp_res; + } + *result= res; + return 0; +} /***************************** LineString *******************************/ uint32 Gis_line_string::get_data_size() const @@ -2162,6 +2275,81 @@ const Geometry::Class_info *Gis_multi_point::get_class_info() const } +/** + Function that calculate spherical distance of Multipoints geometries. + In case there is single point in Multipoint geometries calculate_haversine() + can handle such case. Otherwise, new geometry (Point) has to be constructed. + + @param g pointer to the Geometry + @param r sphere radius + @param result pointer to the result + @param err pointer to the error obtained from calculate_haversin() + + @return state + @retval TRUE failed + @retval FALSE success +*/ +int Gis_multi_point::spherical_distance_multipoints(Geometry *g, const double r, + double *result, int *err) +{ + const uint32 len= 4 + WKB_HEADER_SIZE + POINT_DATA_SIZE + 1; + // Check how many points are stored in Multipoints + uint32 num_of_points1, num_of_points2; + // To find the minimum radius it cannot be greater than Earth radius + double res= 6370986.0; + + /* From Item_func_sphere_distance::spherical_distance_points, + we are sure that there will be multiple points and we have to construct + Point geometry and return the smallest result. + */ + num_geometries(&num_of_points1); + DBUG_ASSERT(num_of_points1 >= 1); + g->num_geometries(&num_of_points2); + DBUG_ASSERT(num_of_points2 >= 1); + + for (uint32 i=1; i <= num_of_points1; i++) + { + Geometry_buffer buff_temp; + Geometry *temp; + double temp_res= 0.0; + char s[len]; + // First 4 bytes are handled already, make sure to create a Point + memset(s + 4, Geometry::wkb_point, 1); + memcpy(s + 5, this->get_data_ptr() + 5, 4); + memcpy(s + 4 + WKB_HEADER_SIZE, this->get_data_ptr() + 4 + WKB_HEADER_SIZE*i +\ + POINT_DATA_SIZE*(i-1), POINT_DATA_SIZE); + s[len-1]= '\0'; + temp= Geometry::construct(&buff_temp, s, len); + DBUG_ASSERT(temp); + // Optimization for single Multipoint + if (num_of_points2 == 1) + { + *result= static_cast(temp)->calculate_haversine(g, r, err); + return 0; + } + for (uint32 j=1; j<= num_of_points2; j++) + { + Geometry_buffer buff_temp2; + Geometry *temp2; + char s2[len]; + // First 4 bytes are handled already, make sure to create a Point + memset(s2 + 4, Geometry::wkb_point, 1); + memcpy(s2 + 5, g->get_data_ptr() + 5, 4); + memcpy(s2 + 4 + WKB_HEADER_SIZE, g->get_data_ptr() + 4 + WKB_HEADER_SIZE*j +\ + POINT_DATA_SIZE*(j-1), POINT_DATA_SIZE); + s2[len-1]= '\0'; + temp2= Geometry::construct(&buff_temp2, s2, len); + DBUG_ASSERT(temp2); + temp_res= static_cast(temp)->calculate_haversine(temp2, r, err); + if (res > temp_res) + res= temp_res; + } + } + *result= res; + return 0; +} + + /***************************** MultiLineString *******************************/ uint32 Gis_multi_line_string::get_data_size() const diff --git a/sql/spatial.h b/sql/spatial.h index e78da6e615d..1da788b07a0 100644 --- a/sql/spatial.h +++ b/sql/spatial.h @@ -332,6 +332,11 @@ public: m_data+= WKB_HEADER_SIZE; } + const char *get_data_ptr() const + { + return m_data; + } + bool envelope(String *result) const; static Class_info *ci_collection[wkb_last+1]; @@ -410,6 +415,17 @@ public: return 0; } + int get_xy_radian(double *x, double *y) const + { + if (!get_xy(x, y)) + { + *x= (*x)*M_PI/180; + *y= (*y)*M_PI/180; + return 0; + } + return 1; + } + int get_x(double *x) const { if (no_data(m_data, SIZEOF_STORED_DOUBLE)) @@ -436,6 +452,10 @@ public: } int store_shapes(Gcalc_shape_transporter *trn) const; const Class_info *get_class_info() const; + double calculate_haversine(const Geometry *g, const double sphere_radius, + int *error); + int spherical_distance_multipoints(Geometry *g, const double r, double *result, + int *error); }; @@ -535,6 +555,8 @@ public: } int store_shapes(Gcalc_shape_transporter *trn) const; const Class_info *get_class_info() const; + int spherical_distance_multipoints(Geometry *g, const double r, double *res, + int *error); }; From 5eda18f0cac6e6520cc1bbeec28c7c653da737df Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Sat, 27 Mar 2021 10:54:57 +0100 Subject: [PATCH 02/18] MDEV-25272: Wrong function name in error messages upon ST_GeomFromGeoJSON call - Invalid function name during ER_WRONG_VALUE_FOR_TYPE and ER_GIS_INVALID_DATA --- mysql-test/r/gis-json.result | 4 ++-- sql/item_geofunc.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mysql-test/r/gis-json.result b/mysql-test/r/gis-json.result index d507a9994ff..e52a7c809c6 100644 --- a/mysql-test/r/gis-json.result +++ b/mysql-test/r/gis-json.result @@ -62,9 +62,9 @@ SELECT st_astext(st_geomfromgeojson('{ "type": "FeatureCollection", "features": st_astext(st_geomfromgeojson('{ "type": "FeatureCollection", "features": [{ "type": "Feature", "geometry": { "type": "Point", "coordinates": [102.0, 0.5] }, "properties": { "prop0": "value0" } }]}')) GEOMETRYCOLLECTION(POINT(102 0.5)) SELECT ST_AsText(ST_GeomFromGeoJSON('{ "type": "Point", "coordinates": [5.3, 15.0, 4.3]}',5)); -ERROR HY000: Incorrect option value: '5' for function ST_GeometryFromJSON +ERROR HY000: Incorrect option value: '5' for function ST_GeomFromGeoJSON SELECT ST_AsText(ST_GeomFromGeoJSON('{ "type": "Point", "coordinates": [5.3, 15.0, 4.3]}',1)); -ERROR 22023: Invalid GIS data provided to function ST_GeometryFromJSON. +ERROR 22023: Invalid GIS data provided to function ST_GeomFromGeoJSON. SELECT ST_AsText(ST_GeomFromGeoJSON('{ "type": "Point", "coordinates": [5.3, 15.0, 4.3]}',2)); ST_AsText(ST_GeomFromGeoJSON('{ "type": "Point", "coordinates": [5.3, 15.0, 4.3]}',2)) POINT(5.3 15) diff --git a/sql/item_geofunc.cc b/sql/item_geofunc.cc index 2b0f67f4ddc..83a9cabf7b2 100644 --- a/sql/item_geofunc.cc +++ b/sql/item_geofunc.cc @@ -145,7 +145,7 @@ String *Item_func_geometry_from_json::val_str(String *str) { String *sv= args[1]->val_str(&tmp_js); my_error(ER_WRONG_VALUE_FOR_TYPE, MYF(0), - "option", sv->c_ptr_safe(), "ST_GeometryFromJSON"); + "option", sv->c_ptr_safe(), "ST_GeomFromGeoJSON"); null_value= 1; return 0; } @@ -182,7 +182,7 @@ String *Item_func_geometry_from_json::val_str(String *str) code= ER_GEOJSON_NOT_CLOSED; break; case Geometry::GEOJ_DIMENSION_NOT_SUPPORTED: - my_error(ER_GIS_INVALID_DATA, MYF(0), "ST_GeometryFromJSON"); + my_error(ER_GIS_INVALID_DATA, MYF(0), "ST_GeomFromGeoJSON"); break; default: report_json_error_ex(js, &je, func_name(), 0, Sql_condition::WARN_LEVEL_WARN); From 96475b78c55141164ecf2719e000d189e4bcbc3d Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Sat, 27 Mar 2021 23:07:31 +0400 Subject: [PATCH 03/18] MDEV-25457 CREATE / DROP PROCEDURE not logged with audit plugin. CREATE/DROP PROCEDIRE/FUNCTION is now treated as DDL. --- mysql-test/suite/plugins/r/server_audit.result | 12 ++++++++++++ mysql-test/suite/plugins/t/server_audit.test | 11 +++++++++++ plugin/server_audit/server_audit.c | 6 +----- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/plugins/r/server_audit.result b/mysql-test/suite/plugins/r/server_audit.result index bbb3a62f9b4..c207744fba5 100644 --- a/mysql-test/suite/plugins/r/server_audit.result +++ b/mysql-test/suite/plugins/r/server_audit.result @@ -140,6 +140,10 @@ select 2; 2 2 drop table t1; +create procedure pr1() insert into test.t1 values ("foo", 42); +create function fn1(i int) returns int deterministic return i+1; +drop procedure pr1; +drop function fn1; set global server_audit_events='query_ddl,query_dml'; create table t1(id int); insert into t1 values (1), (2); @@ -211,6 +215,10 @@ select 2; 2 2 drop table t1; +create procedure pr1() insert into test.t1 values ("foo", 42); +create function fn1(i int) returns int deterministic return i+1; +drop procedure pr1; +drop function fn1; set global server_audit_events='table'; set global server_audit_incl_users='user1'; create user user1@localhost; @@ -394,6 +402,10 @@ TIME,HOSTNAME,root,localhost,ID,ID,WRITE,mysql,roles_mapping, TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'drop user u1, u2, u3',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'create table t1(id int)',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'drop table t1',0 +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'create procedure pr1() insert into test.t1 values ("foo", 42)',0 +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'create function fn1(i int) returns int deterministic return i+1',0 +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'drop procedure pr1',0 +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'drop function fn1',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'create table t1(id int)',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'insert into t1 values (1), (2)',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'select * from t1',0 diff --git a/mysql-test/suite/plugins/t/server_audit.test b/mysql-test/suite/plugins/t/server_audit.test index 6c75c3bf732..382fc781b66 100644 --- a/mysql-test/suite/plugins/t/server_audit.test +++ b/mysql-test/suite/plugins/t/server_audit.test @@ -107,6 +107,12 @@ select 2; /*! select 2*/; /*comment*/ select 2; drop table t1; + +create procedure pr1() insert into test.t1 values ("foo", 42); +create function fn1(i int) returns int deterministic return i+1; +drop procedure pr1; +drop function fn1; + set global server_audit_events='query_ddl,query_dml'; create table t1(id int); insert into t1 values (1), (2); @@ -147,6 +153,11 @@ select * from t1; select 2; drop table t1; +create procedure pr1() insert into test.t1 values ("foo", 42); +create function fn1(i int) returns int deterministic return i+1; +drop procedure pr1; +drop function fn1; + set global server_audit_events='table'; set global server_audit_incl_users='user1'; diff --git a/plugin/server_audit/server_audit.c b/plugin/server_audit/server_audit.c index 9a954365d83..fd19122809a 100644 --- a/plugin/server_audit/server_audit.c +++ b/plugin/server_audit/server_audit.c @@ -15,7 +15,7 @@ #define PLUGIN_VERSION 0x104 -#define PLUGIN_STR_VERSION "1.4.11" +#define PLUGIN_STR_VERSION "1.4.13" #define _my_thread_var loc_thread_var @@ -858,12 +858,8 @@ struct sa_keyword keywords_to_skip[]= struct sa_keyword not_ddl_keywords[]= { - {4, "DROP", &function_word, SQLCOM_QUERY_ADMIN}, - {4, "DROP", &procedure_word, SQLCOM_QUERY_ADMIN}, {4, "DROP", &user_word, SQLCOM_DCL}, {6, "CREATE", &user_word, SQLCOM_DCL}, - {6, "CREATE", &function_word, SQLCOM_QUERY_ADMIN}, - {6, "CREATE", &procedure_word, SQLCOM_QUERY_ADMIN}, {6, "RENAME", &user_word, SQLCOM_DCL}, {0, NULL, 0, SQLCOM_DDL} }; From 2fc76a50229ab29f6524dc08d21ab52b750b2918 Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Sat, 27 Mar 2021 21:02:09 +0100 Subject: [PATCH 04/18] MDEV-13467: Feature request: Support for ST_Distance_Sphere() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - jump to label ‘handle_errors’ can enter to the scope of non-POD ‘Geometry_buffer buffer2’ --- sql/item_geofunc.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/item_geofunc.cc b/sql/item_geofunc.cc index 83a9cabf7b2..0daf9da4a81 100644 --- a/sql/item_geofunc.cc +++ b/sql/item_geofunc.cc @@ -2549,7 +2549,7 @@ double Item_func_sphere_distance::val_real() null_value= (args[0]->null_value || args[1]->null_value); if (null_value) { - goto handle_errors; + return 0; } if (arg_count == 3) @@ -2559,7 +2559,7 @@ double Item_func_sphere_distance::val_real() if (args[2]->null_value) { null_value= true; - goto handle_errors; + return 0; } if (sphere_radius <= 0) { From c0ca3c4ffa45042d608e55326036443e8c4f600d Mon Sep 17 00:00:00 2001 From: Alexey Bychko Date: Fri, 26 Mar 2021 18:01:10 +0700 Subject: [PATCH 05/18] MDEV-25240 minor upgrade does not perform server restart we need to stop server instance on upgrade, but it may be started either by SysV init script or by SystemD. this commit adds `mysql` target to `systemctl stop` call. `mysql` may be the name of initscript or an alias while `mariadb` is a systemd unit file. --- support-files/rpm/server-posttrans.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/support-files/rpm/server-posttrans.sh b/support-files/rpm/server-posttrans.sh index 1406c78a5f5..1525b592735 100644 --- a/support-files/rpm/server-posttrans.sh +++ b/support-files/rpm/server-posttrans.sh @@ -1,10 +1,15 @@ if [ -r %{restart_flag} ] ; then rm %{restart_flag} + # only restart the server if it was alredy running if [ -x /usr/bin/systemctl ] ; then /usr/bin/systemctl daemon-reload > /dev/null 2>&1 - /usr/bin/systemctl try-restart mariadb.service > /dev/null 2>&1 + if [ /usr/bin/systemctl is-active mysql ]; then + /usr/bin/systemctl restart mysql > /dev/null 2>&1 + else + /usr/bin/systemctl try-restart mariadb.service > /dev/null 2>&1 + fi + # not a systemd-enabled environment, use SysV startup script elif %{_sysconfdir}/init.d/mysql status > /dev/null 2>&1; then - # only restart the server if it was alredy running - %{_sysconfdir}/init.d/mysql restart + %{_sysconfdir}/init.d/mysql restart > /dev/null 2>&1 fi fi From 94dea8ef5b9805fe368972c812183ec365084b87 Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Sat, 27 Mar 2021 22:54:18 +0400 Subject: [PATCH 06/18] MDEV-25457 CREATE / DROP PROCEDURE not logged with audit plugin. CREATE/DROP PROCEDIRE/FUNCTION is now treated as DDL. --- mysql-test/suite/plugins/r/server_audit.result | 15 +++++++++++++++ mysql-test/suite/plugins/t/server_audit.test | 10 ++++++++++ plugin/server_audit/server_audit.c | 6 +----- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/plugins/r/server_audit.result b/mysql-test/suite/plugins/r/server_audit.result index 3fce3346f29..b82b38f4f0a 100644 --- a/mysql-test/suite/plugins/r/server_audit.result +++ b/mysql-test/suite/plugins/r/server_audit.result @@ -118,6 +118,7 @@ CREATE USER u1 IDENTIFIED BY 'pwd-123'; GRANT ALL ON sa_db TO u2 IDENTIFIED BY "pwd-321"; SET PASSWORD FOR u1 = PASSWORD('pwd 098'); CREATE USER u3 IDENTIFIED BY ''; +ALTER USER u3 IDENTIFIED BY 'pwd-456'; drop user u1, u2, u3; set global server_audit_events='query_ddl'; create table t1(id int); @@ -139,6 +140,10 @@ select 2; 2 2 drop table t1; +create procedure pr1() insert into test.t1 values ("foo", 42); +create function fn1(i int) returns int deterministic return i+1; +drop procedure pr1; +drop function fn1; set global server_audit_events='query_ddl,query_dml'; create table t1(id int); insert into t1 values (1), (2); @@ -210,6 +215,10 @@ select 2; 2 2 drop table t1; +create procedure pr1() insert into test.t1 values ("foo", 42); +create function fn1(i int) returns int deterministic return i+1; +drop procedure pr1; +drop function fn1; set global server_audit_events='table'; set global server_audit_incl_users='user1'; create user user1@localhost; @@ -382,6 +391,8 @@ TIME,HOSTNAME,root,localhost,ID,ID,WRITE,mysql,proxies_priv, TIME,HOSTNAME,root,localhost,ID,ID,WRITE,mysql,roles_mapping, TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'CREATE USER u3 IDENTIFIED BY *****',0 TIME,HOSTNAME,root,localhost,ID,ID,WRITE,mysql,user, +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'ALTER USER u3 IDENTIFIED BY \'pwd-456\'',0 +TIME,HOSTNAME,root,localhost,ID,ID,WRITE,mysql,user, TIME,HOSTNAME,root,localhost,ID,ID,WRITE,mysql,db, TIME,HOSTNAME,root,localhost,ID,ID,WRITE,mysql,tables_priv, TIME,HOSTNAME,root,localhost,ID,ID,WRITE,mysql,columns_priv, @@ -391,6 +402,10 @@ TIME,HOSTNAME,root,localhost,ID,ID,WRITE,mysql,roles_mapping, TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'drop user u1, u2, u3',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'create table t1(id int)',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'drop table t1',0 +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'create procedure pr1() insert into test.t1 values ("foo", 42)',0 +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'create function fn1(i int) returns int deterministic return i+1',0 +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'drop procedure pr1',0 +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'drop function fn1',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'create table t1(id int)',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'insert into t1 values (1), (2)',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'select * from t1',0 diff --git a/mysql-test/suite/plugins/t/server_audit.test b/mysql-test/suite/plugins/t/server_audit.test index fa5bd7e1349..eba3e5d6770 100644 --- a/mysql-test/suite/plugins/t/server_audit.test +++ b/mysql-test/suite/plugins/t/server_audit.test @@ -95,6 +95,7 @@ CREATE USER u1 IDENTIFIED BY 'pwd-123'; GRANT ALL ON sa_db TO u2 IDENTIFIED BY "pwd-321"; SET PASSWORD FOR u1 = PASSWORD('pwd 098'); CREATE USER u3 IDENTIFIED BY ''; +ALTER USER u3 IDENTIFIED BY 'pwd-456'; drop user u1, u2, u3; set global server_audit_events='query_ddl'; @@ -106,6 +107,11 @@ select 2; /*! select 2*/; /*comment*/ select 2; drop table t1; +create procedure pr1() insert into test.t1 values ("foo", 42); +create function fn1(i int) returns int deterministic return i+1; +drop procedure pr1; +drop function fn1; + set global server_audit_events='query_ddl,query_dml'; create table t1(id int); insert into t1 values (1), (2); @@ -145,6 +151,10 @@ insert into t1 values (1), (2); select * from t1; select 2; drop table t1; +create procedure pr1() insert into test.t1 values ("foo", 42); +create function fn1(i int) returns int deterministic return i+1; +drop procedure pr1; +drop function fn1; set global server_audit_events='table'; set global server_audit_incl_users='user1'; diff --git a/plugin/server_audit/server_audit.c b/plugin/server_audit/server_audit.c index 47878e2e667..666c928d7cc 100644 --- a/plugin/server_audit/server_audit.c +++ b/plugin/server_audit/server_audit.c @@ -15,7 +15,7 @@ #define PLUGIN_VERSION 0x104 -#define PLUGIN_STR_VERSION "1.4.9" +#define PLUGIN_STR_VERSION "1.4.12" #define _my_thread_var loc_thread_var @@ -854,12 +854,8 @@ struct sa_keyword keywords_to_skip[]= struct sa_keyword not_ddl_keywords[]= { - {4, "DROP", &function_word, SQLCOM_QUERY_ADMIN}, - {4, "DROP", &procedure_word, SQLCOM_QUERY_ADMIN}, {4, "DROP", &user_word, SQLCOM_DCL}, {6, "CREATE", &user_word, SQLCOM_DCL}, - {6, "CREATE", &function_word, SQLCOM_QUERY_ADMIN}, - {6, "CREATE", &procedure_word, SQLCOM_QUERY_ADMIN}, {6, "RENAME", &user_word, SQLCOM_DCL}, {0, NULL, 0, SQLCOM_DDL} }; From 4d870b591dc6b96c38687c6e732423b49ad8765b Mon Sep 17 00:00:00 2001 From: Alexey Yurchenko Date: Mon, 15 Mar 2021 14:35:08 +0200 Subject: [PATCH 07/18] Don't pass password to innobackup via command line, use environment instead Signed-off-by: Julius Goryavsky --- scripts/wsrep_sst_mariabackup.sh | 6 +++--- scripts/wsrep_sst_xtrabackup-v2.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 231cf6e3702..4354e341ca5 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -868,10 +868,10 @@ then fi if [ -n "${WSREP_SST_OPT_PSWD:-}" ]; then - INNOEXTRA+=" --password=$WSREP_SST_OPT_PSWD" + export MYSQL_PWD=$WSREP_SST_OPT_PSWD elif [[ $usrst -eq 1 ]];then - # Empty password, used for testing, debugging etc. - INNOEXTRA+=" --password=" + # Empty password, used for testing, debugging etc. + unset MYSQL_PWD fi get_keys diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 8fbbeda170c..8c8af9a0219 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -958,10 +958,10 @@ then fi if [ -n "${WSREP_SST_OPT_PSWD:-}" ]; then - INNOEXTRA+=" --password=$WSREP_SST_OPT_PSWD" + export MYSQL_PWD=$WSREP_SST_OPT_PSWD elif [[ $usrst -eq 1 ]];then - # Empty password, used for testing, debugging etc. - INNOEXTRA+=" --password=" + # Empty password, used for testing, debugging etc. + unset MYSQL_PWD fi get_keys From 85b6a8180599e4d868f43af8c5926f13a1d4dcaa Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 14 Jan 2021 16:55:35 +1100 Subject: [PATCH 08/18] MDEV-24586: remove mysql_to_mariadb.sql This script is unused and unmaintained. The logic is implemented in scripts/mysql_system_tables_fix.sql that forms part of mysql_upgrade Its components: alter table mysql.user drop column `password_last_changed`, drop column `password_lifetime`, drop column `account_locked`; has a friendlier migration path coming MDEV-24122 alter table mysql.user change column `authentication_string` `auth_string` text COLLATE utf8_bin NOT NULL; Already part of scripts/mysql_system_tables_fix.sql alter table mysql.user add column `Password` char(41) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT '' after `user`, add column `is_role` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' after `auth_string`; alter table mysql.user add column `default_role` char(80) COLLATE utf8_bin NOT NULL DEFAULT '', add column `max_statement_time` decimal(12,6) NOT NULL DEFAULT '0.000000'; corrected in MDEV-23201 to be in the right order. update mysql.user set `password`=`auth_string`, plugin='' where plugin="mysql_native_password"; Is handled in server in the function acl_load. --- scripts/CMakeLists.txt | 1 - scripts/mysql_to_mariadb.sql | 22 ---------------------- 2 files changed, 23 deletions(-) delete mode 100644 scripts/mysql_to_mariadb.sql diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index e2fd853ce13..190d9f774fb 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -99,7 +99,6 @@ INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/mysql_test_db.sql ${CMAKE_CURRENT_SOURCE_DIR}/fill_help_tables.sql ${CMAKE_CURRENT_SOURCE_DIR}/mysql_test_data_timezone.sql - ${CMAKE_CURRENT_SOURCE_DIR}/mysql_to_mariadb.sql ${CMAKE_CURRENT_BINARY_DIR}/maria_add_gis_sp.sql ${CMAKE_CURRENT_BINARY_DIR}/maria_add_gis_sp_bootstrap.sql ${FIX_PRIVILEGES_SQL} diff --git a/scripts/mysql_to_mariadb.sql b/scripts/mysql_to_mariadb.sql deleted file mode 100644 index 4ee3f3a4214..00000000000 --- a/scripts/mysql_to_mariadb.sql +++ /dev/null @@ -1,22 +0,0 @@ --- Script that changes MySQL 5.7 privilege tables to MariaDB 10.x --- This should be run first with --- mysql --force mysql < mysql_to_mariadb.sql --- It's ok to ignore any errors, as these usually means that the tables are --- already fixed. - --- After this script s run, one should run at least: --- mysql_upgrade --upgrade-system-tables --- to get the other tables in the mysql database fixed. - --- Drop not existing columnms -alter table mysql.user drop column `password_last_changed`, drop column `password_lifetime`, drop column `account_locked`; - --- Change existing columns -alter table mysql.user change column `authentication_string` `auth_string` text COLLATE utf8_bin NOT NULL; - --- Add new columns -alter table mysql.user add column `Password` char(41) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT '' after `user`, add column `is_role` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N' after `auth_string`; -alter table mysql.user add column `default_role` char(80) COLLATE utf8_bin NOT NULL DEFAULT '', add column `max_statement_time` decimal(12,6) NOT NULL DEFAULT '0.000000'; - --- Fix passwords -update mysql.user set `password`=`auth_string`, plugin='' where plugin="mysql_native_password"; From add24e788971af4a47e54182c6e81fcb53fdbee2 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 29 Mar 2021 14:32:36 +0200 Subject: [PATCH 09/18] Windows - suppress nonsensical(for this OS) system check. Amends 48141f3c1787de941d969ad1e6675611b2b650c2 --- cmake/os/WindowsCache.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/os/WindowsCache.cmake b/cmake/os/WindowsCache.cmake index fe976b96592..c6b334d10a3 100644 --- a/cmake/os/WindowsCache.cmake +++ b/cmake/os/WindowsCache.cmake @@ -94,6 +94,7 @@ SET(HAVE_LRAND48 CACHE INTERNAL "") SET(HAVE_LSTAT CACHE INTERNAL "") SET(HAVE_MADVISE CACHE INTERNAL "") SET(HAVE_MALLINFO CACHE INTERNAL "") +SET(HAVE_MALLINFO2 CACHE INTERNAL "") SET(HAVE_MALLOC_H 1 CACHE INTERNAL "") SET(HAVE_MEMALIGN CACHE INTERNAL "") SET(HAVE_MEMCPY 1 CACHE INTERNAL "") From fb3b2eb5179eda6ff8a4ac799e92e81c30d50aba Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 30 Mar 2021 16:16:24 +1100 Subject: [PATCH 10/18] mallinfo2: whitespace fix --- sql/sql_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/sql_test.cc b/sql/sql_test.cc index d877eb21588..4ad0b365612 100644 --- a/sql/sql_test.cc +++ b/sql/sql_test.cc @@ -619,7 +619,7 @@ Next alarm time: %lu\n", #if defined(HAVE_MALLINFO2) struct mallinfo2 info = mallinfo2(); #elif defined(HAVE_MALLINFO) - struct mallinfo info= mallinfo(); + struct mallinfo info= mallinfo(); #endif #if defined(HAVE_MALLINFO) || defined(HAVE_MALLINFO2) char llbuff[10][22]; From c44273329ed7292e2fdd4039aa4ad20035583f89 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 30 Mar 2021 16:18:30 +1100 Subject: [PATCH 11/18] remove broken tests/grant.pl --- tests/grant.pl | 750 ------------------------------------------------- 1 file changed, 750 deletions(-) delete mode 100755 tests/grant.pl diff --git a/tests/grant.pl b/tests/grant.pl deleted file mode 100755 index b50481a93fc..00000000000 --- a/tests/grant.pl +++ /dev/null @@ -1,750 +0,0 @@ -#!/usr/bin/env perl - -# Copyright (c) 2000, 2005 MySQL AB, 2009 Sun Microsystems, Inc. -# Use is subject to license terms. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA - -# -# Testing of grants. -# Note that this will delete all table and column grants ! -# - -use DBI; -use Getopt::Long; -use strict; - -use vars qw($dbh $user_dbh $opt_help $opt_Information $opt_force $opt_debug - $opt_verbose $opt_server $opt_root_user $opt_password $opt_user - $opt_database $opt_host $version $user $tables_cols $columns_cols - $tmp_table $opt_silent); - -$version="1.1"; -$opt_help=$opt_Information=$opt_force=$opt_debug=$opt_verbose=$opt_silent=0; -$opt_host="localhost", -$opt_server="mysql"; -$opt_root_user="root"; -$opt_password=""; -$opt_user="grant_user"; -$opt_database="grant_test"; - -GetOptions("Information","help","server=s","root-user=s","password=s","user","database=s","force","host=s","debug","verbose","silent") || usage(); -usage() if ($opt_help || $opt_Information); - -$user="$opt_user\@$opt_host"; - -if (!$opt_force) -{ - print_info() -} - -$|=1; - -$tables_cols="Host, Db, User, Table_name, Grantor, Table_priv, Column_priv"; -$columns_cols="Host, Db, User, Table_name, Column_name, Column_priv"; -$tmp_table="/tmp/mysql-grant.test"; # Can't use $$ as we are logging result -unlink($tmp_table); - -# -# clear grant tables -# - -$dbh = DBI->connect("DBI:mysql:mysql:$opt_host", - $opt_root_user,$opt_password, - { PrintError => 0}) || die "Can't connect to mysql server with user '$opt_root_user': $DBI::errstr\n"; - -safe_query("delete from user where user='$opt_user' or user='${opt_user}2'"); -safe_query("delete from db where user='$opt_user'"); -safe_query("delete from tables_priv"); -safe_query("delete from columns_priv"); -safe_query("lock tables mysql.user write"); # Test lock tables -safe_query("flush privileges"); -safe_query("unlock tables"); # should already be unlocked -safe_query("drop database $opt_database",3); # Don't print possible error -safe_query("create database $opt_database"); - -# check that the user can't login yet - -user_connect(1); -#goto test; - -# -# Enable column grant code -# -safe_query("grant select(user) on mysql.user to $user"); -safe_query("revoke select(user) on mysql.user from $user"); - -# -# Test grants on user level -# - -safe_query("grant select on *.* to $user"); -safe_query("set password FOR ${opt_user}2\@$opt_host = password('test')",1); -safe_query("set password FOR $opt_user\@$opt_host=password('test')"); -user_connect(1); -safe_query("set password FOR $opt_user\@$opt_host=''"); -user_connect(0); -user_query("select * from mysql.user where user = '$opt_user'"); -user_query("select * from mysql.db where user = '$opt_user'"); -safe_query("grant select on *.* to $user,$user"); -safe_query("show grants for $user"); -user_connect(0); - -# The following should fail -user_query("insert into mysql.user (host,user) values ('error','$opt_user')",1); -user_query("update mysql.user set host='error' WHERE user='$opt_user'",1); -user_query("create table $opt_database.test (a int,b int)",1); -user_query("grant select on *.* to ${opt_user}2\@$opt_host",1); -safe_query("revoke select on $opt_database.test from $opt_user\@opt_host",1); -safe_query("revoke select on $opt_database.* from $opt_user\@opt_host",1); -safe_query("revoke select on *.* from $opt_user",1); -safe_query("grant select on $opt_database.not_exists to $opt_user",1); -safe_query("grant FILE on $opt_database.test to $opt_user",1); -safe_query("grant select on *.* to wrong___________user_name",1); -safe_query("grant select on $opt_database.* to wrong___________user_name",1); -user_connect(0); -user_query("grant select on $opt_database.test to $opt_user with grant option",1); -safe_query("set password FOR ''\@''=''",1); -user_query("set password FOR root\@$opt_host = password('test')",1); - -# Change privileges for user -safe_query("revoke select on *.* from $user"); -safe_query("grant create,update on *.* to $user"); -user_connect(0); -safe_query("flush privileges"); -user_query("create table $opt_database.test (a int,b int)"); -user_query("update $opt_database.test set b=b+1 where a > 0",1); -safe_query("show grants for $user"); -safe_query("revoke update on *.* from $user"); -user_connect(0); -safe_query("grant select(c) on $opt_database.test to $user",1); -safe_query("revoke select(c) on $opt_database.test from $user",1); -safe_query("grant select on $opt_database.test to wrong___________user_name",1); -user_query("INSERT INTO $opt_database.test values (2,0)",1); - -safe_query("grant ALL PRIVILEGES on *.* to $user"); -safe_query("REVOKE INSERT on *.* from $user"); -user_connect(0); -user_query("INSERT INTO $opt_database.test values (1,0)",1); -safe_query("grant INSERT on *.* to $user"); -user_connect(0); -user_query("INSERT INTO $opt_database.test values (2,0)"); -user_query("select count(*) from $opt_database.test"); -safe_query("revoke SELECT on *.* from $user"); -user_connect(0); -user_query("select count(*) from $opt_database.test",1); -user_query("INSERT INTO $opt_database.test values (3,0)"); -safe_query("grant SELECT on *.* to $user"); -user_connect(0); -user_query("select count(*) from $opt_database.test"); -safe_query("revoke ALL PRIVILEGES on *.* from $user"); -user_connect(1); -safe_query("delete from user where user='$opt_user'"); -safe_query("flush privileges"); -if (0) # Only if no anonymous user on localhost. -{ - safe_query("grant select on *.* to $opt_user"); - user_connect(0); - safe_query("revoke select on *.* from $opt_user"); - user_connect(1); -} -safe_query("delete from user where user='$opt_user'"); -safe_query("flush privileges"); - -# -# Test grants on database level -# -safe_query("grant select on $opt_database.* to $user"); -safe_query("select * from mysql.user where user = '$opt_user'"); -safe_query("select * from mysql.db where user = '$opt_user'"); -user_connect(0); -user_query("select count(*) from $opt_database.test"); -# The following should fail -user_query("select * from mysql.user where user = '$opt_user'",1); -user_query("insert into $opt_database.test values (4,0)",1); -user_query("update $opt_database.test set a=1",1); -user_query("delete from $opt_database.test",1); -user_query("create table $opt_database.test2 (a int)",1); -user_query("ALTER TABLE $opt_database.test add c int",1); -user_query("CREATE INDEX dummy ON $opt_database.test (a)",1); -user_query("drop table $opt_database.test",1); -user_query("grant ALL PRIVILEGES on $opt_database.* to ${opt_user}2\@$opt_host",1); - -# Change privileges for user -safe_query("grant ALL PRIVILEGES on $opt_database.* to $user WITH GRANT OPTION"); -user_connect(0); -user_query("insert into $opt_database.test values (5,0)"); -safe_query("REVOKE ALL PRIVILEGES on * from $user",1); -safe_query("REVOKE ALL PRIVILEGES on *.* from $user"); -safe_query("REVOKE ALL PRIVILEGES on $opt_database.* from $user"); -safe_query("REVOKE ALL PRIVILEGES on $opt_database.* from $user"); -user_connect(0); -user_query("insert into $opt_database.test values (6,0)",1); -safe_query("REVOKE GRANT OPTION on $opt_database.* from $user"); -user_connect(1); -safe_query("grant ALL PRIVILEGES on $opt_database.* to $user"); - -user_connect(0); -user_query("select * from mysql.user where user = '$opt_user'",1); -user_query("insert into $opt_database.test values (7,0)"); -user_query("update $opt_database.test set a=3 where a=2"); -user_query("delete from $opt_database.test where a=3"); -user_query("create table $opt_database.test2 (a int not null)"); -user_query("alter table $opt_database.test2 add b int"); -user_query("create index dummy on $opt_database.test2 (a)"); -user_query("update test,test2 SET test.a=test2.a where test.a=test2.a"); -user_query("drop table $opt_database.test2"); -user_query("show tables from grant_test"); -# These should fail -user_query("insert into mysql.user (host,user) values ('error','$opt_user',0)",1); - -# Revoke database privileges -safe_query("revoke ALL PRIVILEGES on $opt_database.* from $user"); -safe_query("select * from mysql.user where user = '$opt_user'"); -safe_query("select * from mysql.db where user = '$opt_user'"); - -# Test multi-updates -safe_query("grant CREATE,UPDATE,DROP on $opt_database.* to $user"); -user_connect(0); -user_query("create table $opt_database.test2 (a int not null)"); -user_query("update test,test2 SET test.a=1 where 1",1); -user_query("update test,test2 SET test.a=test2.a where 1",1); -safe_query("grant SELECT on $opt_database.* to $user"); -user_connect(0); -user_query("update test,test2 SET test.a=test2.a where test2.a=test.a"); -user_query("drop table $opt_database.test2"); - -# Revoke database privileges -safe_query("revoke ALL PRIVILEGES on $opt_database.* from $user"); -user_connect(1); - -# -# Test of grants on table level -# - -safe_query("grant create on $opt_database.test2 to $user"); -user_connect(0); -user_query("create table $opt_database.test2 (a int not null)"); -user_query("show tables"); # Should only show test, not test2 -user_query("show columns from test",1); -user_query("show keys from test",1); -user_query("show columns from test2"); -user_query("show keys from test2"); -user_query("select * from test",1); -safe_query("grant insert on $opt_database.test to $user"); -user_query("show tables"); -user_query("insert into $opt_database.test values (8,0)"); -user_query("update $opt_database.test set b=1",1); -safe_query("grant update on $opt_database.test to $user"); -user_query("update $opt_database.test set b=2"); - -user_query("update $opt_database.test,test2 SET test.b=3",1); -safe_query("grant select on $opt_database.test2 to $user"); -user_query("update $opt_database.test,test2 SET test.b=3"); -safe_query("revoke select on $opt_database.test2 from $user"); - -user_query("delete from $opt_database.test",1); -safe_query("grant delete on $opt_database.test to $user"); -user_query("delete from $opt_database.test where a=1",1); -user_query("update $opt_database.test set b=3 where b=1",1); -user_query("update $opt_database.test set b=b+1",1); -user_query("update $opt_database.test,test2 SET test.a=test2.a",1); - -# -# Test global SELECT privilege combined with table level privileges -# - -safe_query("grant SELECT on *.* to $user"); -user_connect(0); -user_query("update $opt_database.test set b=b+1"); -user_query("update $opt_database.test set b=b+1 where a > 0"); -user_query("update $opt_database.test,test2 SET test.a=test2.a"); -user_query("update $opt_database.test,test2 SET test2.a=test.a",1); -safe_query("revoke SELECT on *.* from $user"); -safe_query("grant SELECT on $opt_database.* to $user"); -user_connect(0); -user_query("update $opt_database.test set b=b+1"); -user_query("update $opt_database.test set b=b+1 where a > 0"); -safe_query("grant UPDATE on *.* to $user"); -user_connect(0); -user_query("update $opt_database.test set b=b+1"); -user_query("update $opt_database.test set b=b+1 where a > 0"); -safe_query("revoke UPDATE on *.* from $user"); -safe_query("revoke SELECT on $opt_database.* from $user"); -user_connect(0); -user_query("update $opt_database.test set b=b+1 where a > 0",1); -user_query("update $opt_database.test set b=b+1",1); - -# Add one privilege at a time until the user has all privileges -user_query("select * from test",1); -safe_query("grant select on $opt_database.test to $user"); -user_query("delete from $opt_database.test where a=1"); -user_query("update $opt_database.test set b=2 where b=1"); -user_query("update $opt_database.test set b=b+1"); -user_query("select count(*) from test"); -user_query("update test,test2 SET test.b=4",1); -user_query("update test,test2 SET test2.a=test.a",1); -user_query("update test,test2 SET test.a=test2.a",1); - -user_query("create table $opt_database.test3 (a int)",1); -user_query("alter table $opt_database.test2 add c int",1); -safe_query("grant alter on $opt_database.test2 to $user"); -user_query("alter table $opt_database.test2 add c int"); -user_query("create index dummy ON $opt_database.test (a)",1); -safe_query("grant index on $opt_database.test2 to $user"); -user_query("create index dummy ON $opt_database.test2 (a)"); -user_query("insert into test2 SELECT a,a from test",1); -safe_query("grant insert on test2 to $user",1); # No table: mysql.test2 -safe_query("grant insert(a) on $opt_database.test2 to $user"); -user_query("insert into test2 SELECT a,a from test",1); -safe_query("grant insert(c) on $opt_database.test2 to $user"); -user_query("insert into test2 SELECT a,a from test"); -user_query("select count(*) from test2,test",1); -user_query("select count(*) from test,test2",1); -user_query("replace into test2 SELECT a from test",1); -safe_query("grant update on $opt_database.test2 to $user"); -user_query("update test,test2 SET test2.a=test.a"); -user_query("update test,test2 SET test.b=test2.a where 0",1); -user_query("update test,test2 SET test.a=2 where test2.a>100",1); -user_query("update test,test2 SET test.a=test2.a",1); -user_query("replace into test2 SELECT a,a from test",1); -safe_query("grant DELETE on $opt_database.test2 to $user"); -user_query("replace into test2 SELECT a,a from test"); -user_query("insert into test (a) SELECT a from test2",1); -safe_query("grant SELECT on $opt_database.test2 to $user"); -user_query("update test,test2 SET test.b=test2.a where 0"); -user_query("update test,test2 SET test.a=test2.a where test2.a>100"); - -safe_query("revoke UPDATE on $opt_database.test2 from $user"); -safe_query("grant UPDATE (c) on $opt_database.test2 to $user"); -user_query("update test,test2 SET test.b=test2.a where 0"); -user_query("update test,test2 SET test.a=test2.a where test2.a>100"); -user_query("update test,test2 SET test2.a=test2.a where test2.a>100",1); -user_query("update test,test2 SET test2.c=test2.a where test2.a>100"); - -safe_query("revoke SELECT,UPDATE on $opt_database.test2 from $user"); -safe_query("grant UPDATE on $opt_database.test2 to $user"); - -user_query("drop table $opt_database.test2",1); -user_query("grant select on $opt_database.test2 to $user with grant option",1); -safe_query("grant drop on $opt_database.test2 to $user with grant option"); -user_query("grant drop on $opt_database.test2 to $user with grant option"); -user_query("grant select on $opt_database.test2 to $user with grant option",1); - -# check rename privileges -user_query("rename table $opt_database.test2 to $opt_database.test3",1); -safe_query("grant CREATE,DROP on $opt_database.test3 to $user"); -user_query("rename table $opt_database.test2 to $opt_database.test3",1); -user_query("create table $opt_database.test3 (a int)"); -safe_query("grant INSERT on $opt_database.test3 to $user"); -user_query("drop table $opt_database.test3"); -user_query("rename table $opt_database.test2 to $opt_database.test3"); -user_query("rename table $opt_database.test3 to $opt_database.test2",1); -safe_query("grant ALTER on $opt_database.test3 to $user"); -user_query("rename table $opt_database.test3 to $opt_database.test2"); -safe_query("revoke DROP on $opt_database.test2 from $user"); -user_query("rename table $opt_database.test2 to $opt_database.test3"); -user_query("drop table if exists $opt_database.test2,$opt_database.test3",1); -safe_query("drop table if exists $opt_database.test2,$opt_database.test3"); - -# Check that the user doesn't have some user privileges -user_query("create database $opt_database",1); -user_query("drop database $opt_database",1); -user_query("flush tables",1); -safe_query("flush privileges"); - -safe_query("select $tables_cols from mysql.tables_priv"); -safe_query("revoke ALL PRIVILEGES on $opt_database.test from $user"); -safe_query("revoke ALL PRIVILEGES on $opt_database.test2 from $user"); -safe_query("revoke ALL PRIVILEGES on $opt_database.test3 from $user"); -safe_query("revoke GRANT OPTION on $opt_database.test2 from $user"); -safe_query("select $tables_cols from mysql.tables_priv"); -user_query("select count(a) from test",1); - -# -# Test some grants on column level -# - -safe_query("grant create,update on $opt_database.test2 to $user"); -user_query("create table $opt_database.test2 (a int not null)"); -user_query("delete from $opt_database.test where a=2",1); -user_query("delete from $opt_database.test where A=2",1); -user_query("update test set b=5 where b>0",1); -user_query("update test,test2 SET test.b=5 where b>0",1); - -safe_query("grant update(b),delete on $opt_database.test to $user"); -safe_query("revoke update(a) on $opt_database.test from $user",1); -user_query("delete from $opt_database.test where a=2",1); -user_query("update test set b=5 where b>0",1); -safe_query("grant select(a),select(b) on $opt_database.test to $user"); -user_query("delete from $opt_database.test where a=2"); -user_query("delete from $opt_database.test where A=2"); -user_query("update test set b=5 where b>0"); -user_query("update test set a=11 where b>5",1); -user_query("update test,test2 SET test.b=5 where b>0",1); -user_query("update test,test2 SET test.a=11 where b>0",1); -user_query("update test,test2 SET test.b=test2.a where b>0",1); -user_query("update test,test2 SET test.b=11 where test2.a>0",1); -user_query("select a,A from test"); - -safe_query("select $tables_cols from mysql.tables_priv"); -safe_query("revoke ALL PRIVILEGES on $opt_database.test from $user"); -safe_query("select $tables_cols from mysql.tables_priv"); -safe_query("revoke GRANT OPTION on $opt_database.test from $user",1); -safe_query("drop table $opt_database.test2"); -safe_query("revoke create,update on $opt_database.test2 from $user"); - -# -# Test grants on database level -# - -safe_query("grant select(a) on $opt_database.test to $user"); -user_query("show full columns from test"); -safe_query("grant insert (b), update (b) on $opt_database.test to $user"); - -user_query("select count(a) from test"); -user_query("select count(skr.a) from test as skr"); -user_query("select count(a) from test where a > 5"); -user_query("insert into test (b) values (5)"); -user_query("insert into test (b) values (a)"); -user_query("update test set b=3 where a > 0"); - -user_query("select * from test",1); -user_query("select b from test",1); -user_query("select a from test where b > 0",1); -user_query("insert into test (a) values (10)",1); -user_query("insert into test (b) values (b)",1); -user_query("insert into test (a,b) values (1,5)",1); -user_query("insert into test (b) values (1),(b)",1); -user_query("update test set b=3 where b > 0",1); - -safe_query("select $tables_cols from mysql.tables_priv"); -safe_query("select $columns_cols from mysql.columns_priv"); -safe_query("revoke select(a), update (b) on $opt_database.test from $user"); -safe_query("select $tables_cols from mysql.tables_priv"); -safe_query("select $columns_cols from mysql.columns_priv"); - -user_query("select count(a) from test",1); -user_query("update test set b=4",1); - -safe_query("grant select(a,b), update (a,b) on $opt_database.test to $user"); -user_query("select count(a),count(b) from test where a+b > 0"); -user_query("insert into test (b) values (9)"); -user_query("update test set b=6 where b > 0"); - -safe_query("flush privileges"); # Test restoring privileges from disk -safe_query("select $tables_cols from mysql.tables_priv"); -safe_query("select $columns_cols from mysql.columns_priv"); - -# Try mixing of table and database privileges - -user_query("insert into test (a,b) values (12,12)",1); -safe_query("grant insert on $opt_database.* to $user"); -user_connect(0); -user_query("insert into test (a,b) values (13,13)"); - -# This grants and revokes SELECT on different levels. -safe_query("revoke select(b) on $opt_database.test from $user"); -user_query("select count(a) from test where a+b > 0",1); -user_query("update test set b=5 where a=2"); -safe_query("grant select on $opt_database.test to $user"); -user_connect(0); -user_query("select count(a) from test where a+b > 0"); -safe_query("revoke select(b) on $opt_database.test from $user"); -user_query("select count(a) from test where a+b > 0"); -safe_query("revoke select on $opt_database.test from $user"); -user_connect(0); -user_query("select count(a) from test where a+b > 0",1); -safe_query("grant select(a) on $opt_database.test to $user"); -user_query("select count(a) from test where a+b > 0",1); -safe_query("grant select on *.* to $user"); -user_connect(0); -user_query("select count(a) from test where a+b > 0"); -safe_query("revoke select on *.* from $user"); -safe_query("grant select(b) on $opt_database.test to $user"); -user_connect(0); -user_query("select count(a) from test where a+b > 0"); - - -safe_query("select * from mysql.db where user = '$opt_user'"); -safe_query("select $tables_cols from mysql.tables_priv where user = '$opt_user'"); -safe_query("select $columns_cols from mysql.columns_priv where user = '$opt_user'"); - -safe_query("revoke ALL PRIVILEGES on $opt_database.test from $user"); -user_query("select count(a) from test",1); -user_query("select * from mysql.user order by hostname",1); -safe_query("select * from mysql.db where user = '$opt_user'"); -safe_query("select $tables_cols from mysql.tables_priv where user = '$opt_user'"); -safe_query("select $columns_cols from mysql.columns_priv where user = '$opt_user'"); - -# -# Clear up privileges to make future tests easier - -safe_query("delete from user where user='$opt_user'"); -safe_query("delete from db where user='$opt_user'"); -safe_query("flush privileges"); -safe_query("show grants for $user",1); - -# -# Test IDENTIFIED BY -# - -safe_query("grant ALL PRIVILEGES on $opt_database.test to $user identified by 'dummy', ${opt_user}\@127.0.0.1 identified by 'dummy2'"); -user_connect(0,"dummy"); -safe_query("grant SELECT on $opt_database.* to $user identified by ''"); -user_connect(0); -safe_query("revoke ALL PRIVILEGES on $opt_database.test from $user identified by '', ${opt_user}\@127.0.0.1 identified by 'dummy2'"); -safe_query("revoke ALL PRIVILEGES on $opt_database.* from $user identified by ''"); - -safe_query("show grants for $user"); - -# -# Test bug reported in SELECT INTO OUTFILE -# - -safe_query("create table $opt_database.test3 (a int, b int)"); -safe_query("grant SELECT on $opt_database.test3 to $user"); -safe_query("grant FILE on *.* to $user"); -safe_query("insert into $opt_database.test3 values (1,1)"); -user_connect(0); -user_query("select * into outfile '$tmp_table' from $opt_database.test3"); -safe_query("revoke SELECT on $opt_database.test3 from $user"); -safe_query("grant SELECT(a) on $opt_database.test3 to $user"); -user_query("select a from $opt_database.test3"); -user_query("select * from $opt_database.test3",1); -user_query("select a,b from $opt_database.test3",1); -user_query("select b from $opt_database.test3",1); - -safe_query("revoke SELECT(a) on $opt_database.test3 from $user"); -safe_query("revoke FILE on *.* from $user"); -safe_query("drop table $opt_database.test3"); - -# -# Test privileges needed for LOCK TABLES -# - -safe_query("create table $opt_database.test3 (a int)"); -user_connect(1); -safe_query("grant INSERT on $opt_database.test3 to $user"); -user_connect(0); -user_query("select * into outfile '$tmp_table' from $opt_database.test3",1); -safe_query("grant SELECT on $opt_database.test3 to $user"); -user_connect(0); -user_query("LOCK TABLES $opt_database.test3 READ",1); -safe_query("grant LOCK TABLES on *.* to $user"); -safe_query("show grants for $user"); -safe_query("select * from mysql.user where user='$opt_user'"); -user_connect(0); -user_query("LOCK TABLES $opt_database.test3 READ"); -user_query("UNLOCK TABLES"); -safe_query("revoke SELECT,INSERT,UPDATE,DELETE on $opt_database.test3 from $user"); -user_connect(0); -safe_query("revoke LOCK TABLES on *.* from $user"); -user_connect(1); -safe_query("drop table $opt_database.test3"); - -# -# test new privileges in 4.0.2 -# - -safe_query("show grants for $user"); -safe_query("grant all on *.* to $user WITH MAX_QUERIES_PER_HOUR 1 MAX_UPDATES_PER_HOUR 2 MAX_CONNECTIONS_PER_HOUR 3"); -safe_query("show grants for $user"); -safe_query("revoke LOCK TABLES on *.* from $user"); -safe_query("flush privileges"); -safe_query("show grants for $user"); -safe_query("revoke ALL PRIVILEGES on *.* from $user"); -safe_query("show grants for $user"); - -# -# Clean up things -# - -unlink($tmp_table); -safe_query("drop database $opt_database"); -safe_query("delete from user where user='$opt_user'"); -safe_query("delete from db where user='$opt_user'"); -safe_query("delete from tables_priv"); -safe_query("delete from columns_priv"); -safe_query("flush privileges"); - -print "end of test\n"; -exit 0; - -sub usage -{ - print <; chomp($tmp); $tmp=lc($tmp); - last if ($tmp =~ /^yes$/i); - exit 1 if ($tmp =~ /^n/i); - print "\n"; - } -} - - -sub user_connect -{ - my ($ignore_error,$password)=@_; - $password="" if (!defined($password)); - - print "Connecting $opt_user\n" if ($opt_verbose); - $user_dbh->disconnect if (defined($user_dbh)); - - $user_dbh=DBI->connect("DBI:mysql:$opt_database:$opt_host",$opt_user, - $password, { PrintError => 0}); - if (!$user_dbh) - { - if ($opt_verbose || !$ignore_error) - { - print "Error on connect: $DBI::errstr\n"; - } - if (!$ignore_error) - { - die "The above should not have failed!"; - } - } - elsif ($ignore_error) - { - die "Connect succeeded when it shouldn't have !\n"; - } -} - -sub safe_query -{ - my ($query,$ignore_error)=@_; - if (do_query($dbh,$query, $ignore_error)) - { - if (!defined($ignore_error)) - { - die "The above should not have failed!"; - } - } - elsif (defined($ignore_error) && $ignore_error == 1) - { - die "Query '$query' succeeded when it shouldn't have !\n"; - } -} - - -sub user_query -{ - my ($query,$ignore_error)=@_; - if (do_query($user_dbh,$query, $ignore_error)) - { - if (!defined($ignore_error)) - { - die "Query '$query' should not have failed!"; - } - } - elsif (defined($ignore_error) && $ignore_error == 1) - { - die "Query '$query' succeeded when it shouldn't have !\n"; - } -} - - -sub do_query -{ - my ($my_dbh, $query, $ignore_error)=@_; - my ($sth, $row, $tab, $col, $found, $fatal_error); - - print "$query\n" if ($opt_debug || $opt_verbose); - if (!($sth= $my_dbh->prepare($query))) - { - print "Error in prepare: $DBI::errstr\n"; - return 1; - } - if (!$sth->execute) - { - $fatal_error= ($DBI::errstr =~ /parse error/); - if (!$ignore_error || ($opt_verbose && $ignore_error != 3) || $fatal_error) - { - print "Error in execute: $DBI::errstr\n"; - } - die if ($fatal_error); - $sth->finish; - return 1; - } - $found=0; - if (!$opt_silent) - { - while (($row=$sth->fetchrow_arrayref)) - { - $found=1; - $tab=""; - foreach $col (@$row) - { - print $tab; - print defined($col) ? $col : "NULL"; - $tab="\t"; - } - print "\n"; - } - print "\n" if ($found); - } - $sth->finish; - return 0; -} From d217a925b267727936bd0a08dce1863ab6d1ed19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Wed, 3 Mar 2021 12:14:23 +0200 Subject: [PATCH 12/18] MDEV-24923 : Port selected Galera conflict resolution changes from 10.6 Add condition on trx->state == TRX_STATE_COMMITTED_IN_MEMORY in order to avoid unnecessary work. If a transaction has already been committed or rolled back, it will release its locks in lock_release() and let the waiting thread(s) continue execution. Let BF wait on lock_rec_has_to_wait and if necessary other BF is replayed. wsrep_trx_order_before If BF is not even replicated yet then they are ordered correctly. bg_wsrep_kill_trx Make sure victim_trx is found and check also its state. If state is TRX_STATE_COMMITTED_IN_MEMORY transaction is already committed or rolled back and will release it locks soon. wsrep_assert_no_bf_bf_wait Transaction requesting new record lock should be TRX_STATE_ACTIVE Conflicting transaction can be in states TRX_STATE_ACTIVE, TRX_STATE_COMMITTED_IN_MEMORY or in TRX_STATE_PREPARED. If conflicting transaction is already committed in memory or prepared we should wait. When transaction is committed in memory we held trx mutex, but not lock_sys->mutex. Therefore, we could end here before transaction has time to do lock_release() that is protected with lock_sys->mutex. lock_rec_has_to_wait We very well can let bf to wait normally as other BF will be replayed in case of conflict. For debug builds we will do additional sanity checks to catch unsupported bf wait if any. wsrep_kill_victim Check is victim already in TRX_STATE_COMMITTED_IN_MEMORY state and if it is we can return. lock_rec_dequeue_from_page lock_rec_unlock Remove unnecessary wsrep_assert_no_bf_bf_wait function calls. We can very well let BF wait here. --- sql/wsrep_mysqld.cc | 22 ++-- storage/innobase/handler/ha_innodb.cc | 33 +++-- storage/innobase/lock/lock0lock.cc | 175 +++++++++++--------------- 3 files changed, 108 insertions(+), 122 deletions(-) diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc index c033f7e1464..f22d8bf0f5a 100644 --- a/sql/wsrep_mysqld.cc +++ b/sql/wsrep_mysqld.cc @@ -2788,16 +2788,18 @@ extern "C" bool wsrep_thd_ignore_table(THD *thd) extern int wsrep_trx_order_before(THD *thd1, THD *thd2) { - if (wsrep_thd_trx_seqno(thd1) < wsrep_thd_trx_seqno(thd2)) { - WSREP_DEBUG("BF conflict, order: %lld %lld\n", - (long long)wsrep_thd_trx_seqno(thd1), - (long long)wsrep_thd_trx_seqno(thd2)); - return 1; - } - WSREP_DEBUG("waiting for BF, trx order: %lld %lld\n", - (long long)wsrep_thd_trx_seqno(thd1), - (long long)wsrep_thd_trx_seqno(thd2)); - return 0; + const longlong trx1_seqno= wsrep_thd_trx_seqno(thd1); + const longlong trx2_seqno= wsrep_thd_trx_seqno(thd2); + WSREP_DEBUG("BF conflict, order: %lld %lld\n", + trx1_seqno, trx2_seqno); + + if (trx1_seqno == WSREP_SEQNO_UNDEFINED || + trx2_seqno == WSREP_SEQNO_UNDEFINED) + return 1; /* trx is not yet replicated */ + else if (trx1_seqno < trx2_seqno) + return 1; + + return 0; } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index af6fc4d5b48..445103e0550 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -19596,21 +19596,32 @@ static void bg_wsrep_kill_trx( DBUG_ENTER("bg_wsrep_kill_trx"); if (thd) { - victim_trx = thd_to_trx(thd); - lock_mutex_enter(); - trx_mutex_enter(victim_trx); - if (victim_trx->id != arg->trx_id) - { - trx_mutex_exit(victim_trx); - lock_mutex_exit(); + victim_trx= thd_to_trx(thd); + /* Victim trx might not exist e.g. on MDL-conflict. */ + if (victim_trx) { + lock_mutex_enter(); + trx_mutex_enter(victim_trx); + if (victim_trx->id != arg->trx_id || + victim_trx->state == TRX_STATE_COMMITTED_IN_MEMORY) + { + /* Victim was meanwhile rolled back or + committed */ + trx_mutex_exit(victim_trx); + lock_mutex_exit(); + wsrep_thd_UNLOCK(thd); + victim_trx= NULL; + } + } else { + /* find_thread_by_id locked + THD::LOCK_thd_data */ wsrep_thd_UNLOCK(thd); - victim_trx = NULL; } } if (!victim_trx) { - /* it can happen that trx_id was meanwhile rolled back */ - DBUG_PRINT("wsrep", ("no thd for conflicting lock")); + /* Victim trx might not exist (MDL-conflict) or victim + was meanwhile rolled back or committed because of + a KILL statement or a disconnect. */ goto ret; } @@ -19766,7 +19777,7 @@ static void bg_wsrep_kill_trx( } ret_awake: - awake = true; + awake= true; ret_unlock: trx_mutex_exit(victim_trx); diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 4e5c8fbf0ec..506106a2269 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -644,71 +644,81 @@ lock_rec_get_insert_intention( return(lock->type_mode & LOCK_INSERT_INTENTION); } +#ifdef UNIV_DEBUG #ifdef WITH_WSREP -/** Check if both conflicting lock and other record lock are brute force -(BF). This case is a bug so report lock information and wsrep state. -@param[in] lock_rec1 conflicting waiting record lock or NULL -@param[in] lock_rec2 other waiting record lock -@param[in] trx1 lock_rec1 can be NULL, trx +/** Check if both conflicting lock transaction and other transaction +requesting record lock are brute force (BF). If they are check is +this BF-BF wait correct and if not report BF wait and assert. + +@param[in] lock_rec other waiting record lock +@param[in] trx trx requesting conflicting record lock */ -static void wsrep_assert_no_bf_bf_wait( - const lock_t* lock_rec1, - const lock_t* lock_rec2, - const trx_t* trx1) +static void wsrep_assert_no_bf_bf_wait(const lock_t *lock, const trx_t *trx) { - ut_ad(!lock_rec1 || lock_get_type_low(lock_rec1) == LOCK_REC); - ut_ad(lock_get_type_low(lock_rec2) == LOCK_REC); + ut_ad(lock_get_type_low(lock) == LOCK_REC); ut_ad(lock_mutex_own()); + trx_t* lock_trx= lock->trx; /* Note that we are holding lock_sys->mutex, thus we should not acquire THD::LOCK_thd_data mutex below to avoid mutexing order violation. */ - if (!trx1->is_wsrep() || !lock_rec2->trx->is_wsrep()) + if (!trx->is_wsrep() || !lock_trx->is_wsrep()) return; - if (UNIV_LIKELY(!wsrep_thd_is_BF(trx1->mysql_thd, FALSE))) - return; - if (UNIV_LIKELY(!wsrep_thd_is_BF(lock_rec2->trx->mysql_thd, FALSE))) + if (UNIV_LIKELY(!wsrep_thd_is_BF(trx->mysql_thd, FALSE)) + || UNIV_LIKELY(!wsrep_thd_is_BF(lock_trx->mysql_thd, FALSE))) return; - /* if BF - BF order is honored, we can keep trx1 waiting for the lock */ - if (wsrep_trx_order_before(trx1->mysql_thd, lock_rec2->trx->mysql_thd)) - return; + ut_ad(trx->state == TRX_STATE_ACTIVE); - /* avoiding BF-BF conflict assert, if victim is already aborting - or rolling back for replaying - */ - if (wsrep_trx_is_aborting(lock_rec2->trx->mysql_thd)) + trx_mutex_enter(lock_trx); + const trx_state_t trx2_state= lock_trx->state; + trx_mutex_exit(lock_trx); + + /* If transaction is already committed in memory or + prepared we should wait. When transaction is committed in + memory we held trx mutex, but not lock_sys->mutex. Therefore, + we could end here before transaction has time to do + lock_release() that is protected with lock_sys->mutex. */ + switch (trx2_state) { + case TRX_STATE_COMMITTED_IN_MEMORY: + case TRX_STATE_PREPARED: + return; + case TRX_STATE_ACTIVE: + break; + default: + ut_ad("invalid state" == 0); + } + + /* If BF - BF order is honored, i.e. trx already holding + record lock should be ordered before this new lock request + we can keep trx waiting for the lock. If conflicting + transaction is already aborting or rolling back for replaying + we can also let new transaction waiting. */ + if (wsrep_trx_order_before(lock_trx->mysql_thd, trx->mysql_thd) + || wsrep_trx_is_aborting(lock_trx->mysql_thd)) return; mtr_t mtr; - if (lock_rec1) { - ib::error() << "Waiting lock on table: " - << lock_rec1->index->table->name - << " index: " - << lock_rec1->index->name() - << " that has conflicting lock "; - lock_rec_print(stderr, lock_rec1, mtr); - } - ib::error() << "Conflicting lock on table: " - << lock_rec2->index->table->name + << lock->index->table->name << " index: " - << lock_rec2->index->name() + << lock->index->name() << " that has lock "; - lock_rec_print(stderr, lock_rec2, mtr); + lock_rec_print(stderr, lock, mtr); ib::error() << "WSREP state: "; - wsrep_report_bf_lock_wait(trx1->mysql_thd, - trx1->id); - wsrep_report_bf_lock_wait(lock_rec2->trx->mysql_thd, - lock_rec2->trx->id); + wsrep_report_bf_lock_wait(trx->mysql_thd, + trx->id); + wsrep_report_bf_lock_wait(lock_trx->mysql_thd, + lock_trx->id); /* BF-BF wait is a bug */ ut_error; } #endif /* WITH_WSREP */ +#endif /* UNIV_DEBUG */ /*********************************************************************//** Checks if a lock request for a new lock has to wait for request lock2. @@ -832,9 +842,11 @@ lock_rec_has_to_wait( return (FALSE); } - /* There should not be two conflicting locks that are - brute force. If there is it is a bug. */ - wsrep_assert_no_bf_bf_wait(NULL, lock2, trx); + /* We very well can let bf to wait normally as other + BF will be replayed in case of conflict. For debug + builds we will do additional sanity checks to catch + unsupported bf wait if any. */ + ut_d(wsrep_assert_no_bf_bf_wait(lock2, trx)); #endif /* WITH_WSREP */ return(TRUE); @@ -1111,66 +1123,35 @@ lock_rec_other_has_expl_req( #endif /* UNIV_DEBUG */ #ifdef WITH_WSREP -static -void -wsrep_kill_victim( -/*==============*/ - const trx_t * const trx, - const lock_t *lock) +static void wsrep_kill_victim(const trx_t * const trx, const lock_t *lock) { ut_ad(lock_mutex_own()); - ut_ad(trx_mutex_own(lock->trx)); + ut_ad(trx->is_wsrep()); + trx_t* lock_trx = lock->trx; + ut_ad(trx_mutex_own(lock_trx)); + ut_ad(lock_trx != trx); - /* quit for native mysql */ - if (!trx->is_wsrep()) return; + if (!wsrep_thd_is_BF(trx->mysql_thd, FALSE)) + return; - my_bool bf_this = wsrep_thd_is_BF(trx->mysql_thd, FALSE); - my_bool bf_other = wsrep_thd_is_BF(lock->trx->mysql_thd, FALSE); - mtr_t mtr; + if (lock_trx->state == TRX_STATE_COMMITTED_IN_MEMORY + || lock_trx->lock.was_chosen_as_deadlock_victim) + return; - if ((bf_this && !bf_other) || - (bf_this && bf_other && wsrep_trx_order_before( - trx->mysql_thd, lock->trx->mysql_thd))) { + my_bool bf_other = wsrep_thd_is_BF(lock_trx->mysql_thd, FALSE); - if (lock->trx->lock.que_state == TRX_QUE_LOCK_WAIT) { - if (UNIV_UNLIKELY(wsrep_debug)) { - ib::info() << "WSREP: BF victim waiting\n"; - } + if (!bf_other + || wsrep_trx_order_before(trx->mysql_thd, + lock_trx->mysql_thd)) { + + if (lock_trx->lock.que_state == TRX_QUE_LOCK_WAIT) { + if (UNIV_UNLIKELY(wsrep_debug)) + WSREP_INFO("BF victim waiting"); /* cannot release lock, until our lock is in the queue*/ - } else if (lock->trx != trx) { - if (wsrep_log_conflicts) { - if (bf_this) { - ib::info() << "*** Priority TRANSACTION:"; - } else { - ib::info() << "*** Victim TRANSACTION:"; - } - - wsrep_trx_print_locking(stderr, trx, 3000); - - if (bf_other) { - ib::info() << "*** Priority TRANSACTION:"; - } else { - ib::info() << "*** Victim TRANSACTION:"; - } - wsrep_trx_print_locking(stderr, lock->trx, 3000); - - ib::info() << "*** WAITING FOR THIS LOCK TO BE GRANTED:"; - - if (lock_get_type(lock) == LOCK_REC) { - lock_rec_print(stderr, lock, mtr); - } else { - lock_table_print(stderr, lock); - } - - ib::info() << " SQL1: " - << wsrep_thd_query(trx->mysql_thd); - ib::info() << " SQL2: " - << wsrep_thd_query(lock->trx->mysql_thd); - } - - wsrep_innobase_kill_one_trx(trx->mysql_thd, - trx, lock->trx, TRUE); + } else { + wsrep_innobase_kill_one_trx(trx->mysql_thd, trx, + lock_trx, true); } } } @@ -2410,10 +2391,6 @@ static void lock_rec_dequeue_from_page(lock_t* in_lock) /* Grant the lock */ ut_ad(lock->trx != in_lock->trx); lock_grant(lock); -#ifdef WITH_WSREP - } else { - wsrep_assert_no_bf_bf_wait(c, lock, c->trx); -#endif /* WITH_WSREP */ } } } else { @@ -4342,10 +4319,6 @@ released: /* Grant the lock */ ut_ad(trx != lock->trx); lock_grant(lock); -#ifdef WITH_WSREP - } else { - wsrep_assert_no_bf_bf_wait(c, lock, c->trx); -#endif /* WITH_WSREP */ } } } else { From dfda1c9283b82f97e8050169525bcca6848c518d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 30 Mar 2021 08:08:21 +0300 Subject: [PATCH 13/18] Add supression for warning. --- mysql-test/suite/galera/r/galera_ssl_upgrade.result | 1 + mysql-test/suite/galera/t/galera_ssl_upgrade.test | 2 ++ 2 files changed, 3 insertions(+) diff --git a/mysql-test/suite/galera/r/galera_ssl_upgrade.result b/mysql-test/suite/galera/r/galera_ssl_upgrade.result index 818469bf6ad..8aab135c6a2 100644 --- a/mysql-test/suite/galera/r/galera_ssl_upgrade.result +++ b/mysql-test/suite/galera/r/galera_ssl_upgrade.result @@ -1,3 +1,4 @@ +call mtr.add_suppression("WSREP: write_handler(): protocol is shutdown"); SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'; VARIABLE_VALUE = 'Synced' 1 diff --git a/mysql-test/suite/galera/t/galera_ssl_upgrade.test b/mysql-test/suite/galera/t/galera_ssl_upgrade.test index b69e390533f..33c5a43df9d 100644 --- a/mysql-test/suite/galera/t/galera_ssl_upgrade.test +++ b/mysql-test/suite/galera/t/galera_ssl_upgrade.test @@ -8,6 +8,8 @@ --source include/have_innodb.inc --source include/have_ssl_communication.inc +call mtr.add_suppression("WSREP: write_handler(): protocol is shutdown"); + SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'; SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; From c468d5cb50a8a638ff3e9af6f373951ddb68f56a Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 30 Mar 2021 15:18:06 +0530 Subject: [PATCH 14/18] MDEV-15527 page_compressed compressed page partially during import tablespace - Importing table operation fails to punch the hole in the filesystem when page compressed table is involved. To achieve that, InnoDB firstly punches the hole for the IOBuffer size(1MB). After that, InnoDB should write page by page when page compression is involved. --- storage/innobase/row/row0import.cc | 85 ++++++++++++++++++++++++++---- 1 file changed, 74 insertions(+), 11 deletions(-) diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 981c5046de1..2e14455568d 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2020, MariaDB Corporation. +Copyright (c) 2015, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -3375,6 +3375,57 @@ struct fil_iterator_t { byte* crypt_io_buffer; /*!< IO buffer when encrypted */ }; + +/** InnoDB writes page by page when there is page compressed +tablespace involved. It does help to save the disk space when +punch hole is enabled +@param iter Tablespace iterator +@param write_request Request to write into the file +@param offset offset of the file to be written +@param writeptr buffer to be written +@param n_bytes number of bytes to be written +@param try_punch_only Try the range punch only because the + current range is full of empty pages +@return DB_SUCCESS */ +static +dberr_t fil_import_compress_fwrite(const fil_iterator_t &iter, + const IORequest &write_request, + ulint offset, + const byte *writeptr, + ulint n_bytes, + bool try_punch_only=false) +{ + dberr_t err= os_file_punch_hole(iter.file, offset, n_bytes); + if (err != DB_SUCCESS || try_punch_only) + return err; + + for (ulint j= 0; j < n_bytes; j+= srv_page_size) + { + /* Read the original data length from block and + safer to read FIL_PAGE_COMPRESSED_SIZE because it + is not encrypted*/ + ulint n_write_bytes= srv_page_size; + if (j || offset) + { + n_write_bytes= mach_read_from_2(writeptr + j + FIL_PAGE_DATA); + const unsigned ptype= mach_read_from_2(writeptr + j + FIL_PAGE_TYPE); + /* Ignore the empty page */ + if (ptype == 0 && n_write_bytes == 0) + continue; + n_write_bytes+= FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; + if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) + n_write_bytes+= FIL_PAGE_COMPRESSION_METHOD_SIZE; + } + + err= os_file_write(write_request, iter.filepath, iter.file, + writeptr + j, offset + j, n_write_bytes); + if (err != DB_SUCCESS) + break; + } + + return err; +} + /********************************************************************//** TODO: This can be made parallel trivially by chunking up the file and creating a callback per thread. . Main benefit will be to use multiple CPUs for @@ -3416,7 +3467,10 @@ fil_iterate( /* TODO: For ROW_FORMAT=COMPRESSED tables we do a lot of useless copying for non-index pages. Unfortunately, it is required by buf_zip_decompress() */ - dberr_t err = DB_SUCCESS; + dberr_t err = DB_SUCCESS; + bool page_compressed = false; + bool punch_hole = true; + IORequest write_request(IORequest::WRITE); for (offset = iter.start; offset < iter.end; offset += n_bytes) { if (callback.is_interrupted()) { @@ -3494,9 +3548,8 @@ page_corrupted: goto func_exit; } - const bool page_compressed - = fil_page_is_compressed_encrypted(src) - || fil_page_is_compressed(src); + page_compressed= fil_page_is_compressed_encrypted(src) + || fil_page_is_compressed(src); if (page_compressed && block->page.zip.data) { goto page_corrupted; @@ -3651,13 +3704,23 @@ not_encrypted: } } - /* A page was updated in the set, write back to disk. */ - if (updated) { - IORequest write_request(IORequest::WRITE); + if (page_compressed && punch_hole && srv_use_trim) { + err = fil_import_compress_fwrite( + iter, write_request, offset, writeptr, n_bytes, + !updated); - err = os_file_write(write_request, - iter.filepath, iter.file, - writeptr, offset, n_bytes); + if (err != DB_SUCCESS) { + punch_hole = false; + if (updated) { + goto normal_write; + } + } + } else if (updated) { + /* A page was updated in the set, write back to disk. */ +normal_write: + err = os_file_write( + write_request, iter.filepath, iter.file, + writeptr, offset, n_bytes); if (err != DB_SUCCESS) { goto func_exit; From 7c423c26d9c0a676334885fd3f32a486ecbad3af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 30 Mar 2021 16:14:19 +0300 Subject: [PATCH 15/18] Add missing have_perfschema.inc --- mysql-test/suite/binlog/t/binlog_xa_recover.test | 1 + mysql-test/t/userstat.test | 1 + 2 files changed, 2 insertions(+) diff --git a/mysql-test/suite/binlog/t/binlog_xa_recover.test b/mysql-test/suite/binlog/t/binlog_xa_recover.test index 51c30ee705d..3b2a7e45392 100644 --- a/mysql-test/suite/binlog/t/binlog_xa_recover.test +++ b/mysql-test/suite/binlog/t/binlog_xa_recover.test @@ -8,6 +8,7 @@ --source include/have_debug.inc --source include/have_debug_sync.inc --source include/have_binlog_format_row.inc +--source include/have_perfschema.inc # Valgrind does not work well with test that crashes the server --source include/not_valgrind.inc diff --git a/mysql-test/t/userstat.test b/mysql-test/t/userstat.test index 42fe1c2ad17..cc7ddd58e11 100644 --- a/mysql-test/t/userstat.test +++ b/mysql-test/t/userstat.test @@ -5,6 +5,7 @@ -- source include/have_innodb.inc -- source include/have_log_bin.inc +-- source include/have_perfschema.inc select variable_value from information_schema.global_status where variable_name="handler_read_key" into @global_read_key; show columns from information_schema.client_statistics; From 108ba4c380662bb32555477d7b46b5bdfa54c395 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 30 Mar 2021 20:34:39 +0530 Subject: [PATCH 16/18] MDEV-15527 page_compressed compressed page partially during import tablespace - Post push to address 32-bit build failure. --- storage/innobase/row/row0import.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 2e14455568d..fa7db1e27b8 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -3390,7 +3390,7 @@ punch hole is enabled static dberr_t fil_import_compress_fwrite(const fil_iterator_t &iter, const IORequest &write_request, - ulint offset, + os_offset_t offset, const byte *writeptr, ulint n_bytes, bool try_punch_only=false) From b771ab242baf238463837b9d19eb59d544bff475 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 29 Mar 2021 18:51:36 +0530 Subject: [PATCH 17/18] MDEV-25200 Index count mismatch due to aborted FULLTEXT INDEX - Aborting of fulltext index creation fails to remove the index from sys indexes table. When we try to reload the table definition, InnoDB fails with index count mismatch error. InnoDB should remove the index from sys indexes while rollbacking the secondary index creation. --- .../suite/innodb_fts/r/misc_debug.result | 26 ++++++++++++++++ mysql-test/suite/innodb_fts/t/misc_debug.test | 31 +++++++++++++++++++ storage/innobase/row/row0merge.cc | 2 ++ 3 files changed, 59 insertions(+) diff --git a/mysql-test/suite/innodb_fts/r/misc_debug.result b/mysql-test/suite/innodb_fts/r/misc_debug.result index 8ef2ac425fc..b162b2f7415 100644 --- a/mysql-test/suite/innodb_fts/r/misc_debug.result +++ b/mysql-test/suite/innodb_fts/r/misc_debug.result @@ -26,3 +26,29 @@ SET DEBUG_DBUG="+d,fts_instrument_sync"; INSERT INTO t1 VALUES(1, "mariadb"); ALTER TABLE t1 FORCE; DROP TABLE t2, t1; +# +# MDEV-25200 Index count mismatch due to aborted FULLTEXT INDEX +# +CREATE TABLE t1(a INT, b TEXT, c TEXT, FULLTEXT INDEX(b)) ENGINE=InnoDB; +connect con1,localhost,root,,test; +SET DEBUG_SYNC='innodb_inplace_alter_table_enter SIGNAL s1 WAIT_FOR g1'; +SET DEBUG_SYNC='innodb_commit_inplace_alter_table_enter SIGNAL s2 WAIT_FOR g2'; +ALTER TABLE t1 ADD FULLTEXT(c); +connection default; +SET DEBUG_SYNC='now WAIT_FOR s1'; +KILL QUERY @id; +SET DEBUG_SYNC='now SIGNAL g1 WAIT_FOR s2'; +START TRANSACTION; +SELECT * FROM t1; +a b c +SET DEBUG_SYNC='now SIGNAL s2'; +connection con1; +ERROR 70100: Query execution was interrupted +disconnect con1; +connection default; +SET DEBUG_SYNC=RESET; +ALTER TABLE t1 ADD bl INT AS (LENGTH(b)) VIRTUAL; +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check status OK +DROP TABLE t1; diff --git a/mysql-test/suite/innodb_fts/t/misc_debug.test b/mysql-test/suite/innodb_fts/t/misc_debug.test index aaf628abe6d..461e3f1d9d4 100644 --- a/mysql-test/suite/innodb_fts/t/misc_debug.test +++ b/mysql-test/suite/innodb_fts/t/misc_debug.test @@ -5,6 +5,8 @@ --source include/have_innodb.inc --source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/count_sessions.inc # Following test is for Bug 14668777 - ASSERT ON IB_VECTOR_SIZE( # TABLE->FTS->INDEXES, ALTER TABLE @@ -52,3 +54,32 @@ INSERT INTO t1 VALUES(1, "mariadb"); ALTER TABLE t1 FORCE; # Cleanup DROP TABLE t2, t1; + +--echo # +--echo # MDEV-25200 Index count mismatch due to aborted FULLTEXT INDEX +--echo # +CREATE TABLE t1(a INT, b TEXT, c TEXT, FULLTEXT INDEX(b)) ENGINE=InnoDB; +connect(con1,localhost,root,,test); +let $ID= `SELECT @id := CONNECTION_ID()`; +SET DEBUG_SYNC='innodb_inplace_alter_table_enter SIGNAL s1 WAIT_FOR g1'; +SET DEBUG_SYNC='innodb_commit_inplace_alter_table_enter SIGNAL s2 WAIT_FOR g2'; +send ALTER TABLE t1 ADD FULLTEXT(c); +connection default; +SET DEBUG_SYNC='now WAIT_FOR s1'; +let $ignore= `SELECT @id := $ID`; +KILL QUERY @id; +SET DEBUG_SYNC='now SIGNAL g1 WAIT_FOR s2'; +START TRANSACTION; +SELECT * FROM t1; +SET DEBUG_SYNC='now SIGNAL s2'; +connection con1; +--error ER_QUERY_INTERRUPTED +reap; +disconnect con1; +connection default; +SET DEBUG_SYNC=RESET; +# Exploit MDEV-17468 to force the table definition to be reloaded +ALTER TABLE t1 ADD bl INT AS (LENGTH(b)) VIRTUAL; +CHECK TABLE t1; +DROP TABLE t1; +--source include/wait_until_count_sessions.inc diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 3de85f024a3..835d74043fd 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -3768,6 +3768,8 @@ row_merge_drop_indexes( ut_ad(prev); ut_a(table->fts); fts_drop_index(table, index, trx); + row_merge_drop_index_dict( + trx, index->id); /* We can remove a DICT_FTS index from the cache, because we do not allow ADD FULLTEXT INDEX From 99945d77d748470fbb3939be5f7d0e36d24d8097 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Tue, 30 Mar 2021 09:24:25 +0100 Subject: [PATCH 18/18] MDEV-25294 signal handler display coredump on mac --- sql/signal_handler.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/sql/signal_handler.cc b/sql/signal_handler.cc index c913a0477d7..61c53e6b6e3 100644 --- a/sql/signal_handler.cc +++ b/sql/signal_handler.cc @@ -30,6 +30,11 @@ #define SIGNAL_FMT "signal %d" #endif + +#ifdef __APPLE__ +#include +#endif + #ifndef PATH_MAX #define PATH_MAX 4096 #endif @@ -51,7 +56,7 @@ extern const char *optimizer_switch_names[]; static inline void output_core_info() { /* proc is optional on some BSDs so it can't hurt to look */ -#ifdef HAVE_READLINK +#if defined(HAVE_READLINK) && !defined(__APPLE__) char buff[PATH_MAX]; ssize_t len; int fd; @@ -77,6 +82,13 @@ static inline void output_core_info() my_close(fd, MYF(0)); } #endif +#elif defined(__APPLE__) + char buff[PATH_MAX]; + size_t len = sizeof(buff); + if (sysctlbyname("kern.corefile", buff, &len, NULL, 0) == 0) + { + my_safe_printf_stderr("Core pattern: %.*s\n", (int) len, buff); + } #else char buff[80]; my_getwd(buff, sizeof(buff), 0);