From c7d3f24381b2326d1207046699e0810d0ba34303 Mon Sep 17 00:00:00 2001
From: Aleksey Midenkov <midenok@gmail.com>
Date: Thu, 9 Jan 2025 17:11:19 +0300
Subject: [PATCH] MDEV-24726 Assertion `0' failed in
 Field_varstring_compressed::key_cmp

Compressed fields cannot have keys and therefore we are not adding
keys in derived preparation.
---
 mysql-test/main/column_compression.result | 21 +++++++++++++++++++++
 mysql-test/main/column_compression.test   | 23 +++++++++++++++++++++++
 sql/sql_select.cc                         |  4 +++-
 3 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/mysql-test/main/column_compression.result b/mysql-test/main/column_compression.result
index 45e073b72b1..0d312a71546 100644
--- a/mysql-test/main/column_compression.result
+++ b/mysql-test/main/column_compression.result
@@ -2978,4 +2978,25 @@ SELECT GROUP_CONCAT( b, a ORDER BY 2 ) AS f FROM t1;
 f
 nc,mmmmmmmmmmd
 DROP TABLE t1;
+#
+# MDEV-24726 Assertion `0' failed in Field_varstring_compressed::key_cmp
+#
+CREATE TABLE t1 (a VARCHAR(8) COMPRESSED) CHARACTER SET utf8mb4;
+CREATE ALGORITHM=TEMPTABLE VIEW v1 AS SELECT * FROM t1;
+INSERT INTO t1 VALUES ('foo'),('bar'),('foo');
+SELECT * FROM v1 WHERE a IN (SELECT a FROM t1);
+a
+foo
+bar
+foo
+DROP VIEW v1;
+DROP TABLE t1;
+CREATE TABLE t1 (f1 VARCHAR(8)) CHARSET=eucjpms COLLATE=eucjpms_nopad_bin;
+INSERT INTO t1 VALUES ('');
+CREATE TABLE t2 (f2 VARCHAR(8) COMPRESSED) CHARSET=eucjpms COLLATE=eucjpms_nopad_bin;
+INSERT INTO t2 VALUES ('a'),('b');
+SELECT t1.* FROM t1 LEFT JOIN (SELECT DISTINCT f2 FROM t2) sq ON sq.f2 = t1.f1;
+f1
+
+DROP TABLE t1, t2;
 # End of 10.5 tests
diff --git a/mysql-test/main/column_compression.test b/mysql-test/main/column_compression.test
index dffa42e2017..68512aaacf4 100644
--- a/mysql-test/main/column_compression.test
+++ b/mysql-test/main/column_compression.test
@@ -517,4 +517,27 @@ INSERT INTO t1 VALUES ('c','n'),('d','mmmmmmmmmm');
 SELECT GROUP_CONCAT( b, a ORDER BY 2 ) AS f FROM t1;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-24726 Assertion `0' failed in Field_varstring_compressed::key_cmp
+--echo #
+
+CREATE TABLE t1 (a VARCHAR(8) COMPRESSED) CHARACTER SET utf8mb4;
+CREATE ALGORITHM=TEMPTABLE VIEW v1 AS SELECT * FROM t1;
+INSERT INTO t1 VALUES ('foo'),('bar'),('foo');
+
+SELECT * FROM v1 WHERE a IN (SELECT a FROM t1);
+
+# Cleanup
+DROP VIEW v1;
+DROP TABLE t1;
+
+CREATE TABLE t1 (f1 VARCHAR(8)) CHARSET=eucjpms COLLATE=eucjpms_nopad_bin;
+INSERT INTO t1 VALUES ('');
+CREATE TABLE t2 (f2 VARCHAR(8) COMPRESSED) CHARSET=eucjpms COLLATE=eucjpms_nopad_bin;
+INSERT INTO t2 VALUES ('a'),('b');
+SELECT t1.* FROM t1 LEFT JOIN (SELECT DISTINCT f2 FROM t2) sq ON sq.f2 = t1.f1;
+
+# Cleanup
+DROP TABLE t1, t2;
+
 --echo # End of 10.5 tests
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index f0da265f86b..1b6686aee58 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -6751,7 +6751,9 @@ add_key_part(DYNAMIC_ARRAY *keyuse_array, KEY_FIELD *key_field)
 	}
       }
     }
-    if (field->hash_join_is_possible() &&
+    /* Compressed fields can't have keys (see Field_varstring_compressed::key_cmp()). */
+    if (!field->compression_method() &&
+        field->hash_join_is_possible() &&
         (key_field->optimize & KEY_OPTIMIZE_EQ) &&
         key_field->val->used_tables())
     {