MDEV-27149: Add rocksdb_ignore_datadic_errors

Add a --rocksdb_ignore_datadic_errors plugin option for MyRocks.

The default is 0, and this means MyRocks will call abort() if it detects
a DDL mismatch.

Setting rocksdb_ignore_datadic_errors=1 makes MyRocks to try to ignore the
errors and allow to start the server for repairs.

storage/rocksdb/ha_rocksdb.cc

@@ -637,6 +637,8 @@ static my_bool rocksdb_large_prefix = 0;
 static my_bool rocksdb_allow_to_start_after_corruption = 0;
 static char* rocksdb_git_hash;
 
+uint32_t rocksdb_ignore_datadic_errors = 0;
+
 char *compression_types_val=
   const_cast<char*>(get_rocksdb_supported_compression_types());
 static unsigned long rocksdb_write_policy =
@@ -1907,6 +1909,15 @@ static MYSQL_SYSVAR_UINT(
     nullptr, nullptr, 1 /* default value */, 0 /* min value */,
     2 /* max value */, 0);
 
+static MYSQL_SYSVAR_UINT(
+    ignore_datadic_errors, rocksdb_ignore_datadic_errors,
+    PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
+    "Ignore MyRocks' data directory errors. "
+    "(CAUTION: Use only to start the server and perform repairs. Do NOT use "
+    "for regular operation)",
+    nullptr, nullptr, 0 /* default value */, 0 /* min value */,
+    1 /* max value */, 0);
+
 static MYSQL_SYSVAR_STR(datadir, rocksdb_datadir,
                         PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY,
                         "RocksDB data directory", nullptr, nullptr,
@@ -2142,6 +2153,8 @@ static struct st_mysql_sys_var *rocksdb_system_variables[] = {
     MYSQL_SYSVAR(rollback_on_timeout),
 
     MYSQL_SYSVAR(enable_insert_with_update_caching),
+
+    MYSQL_SYSVAR(ignore_datadic_errors),
     nullptr};
 
 static rocksdb::WriteOptions rdb_get_rocksdb_write_options(
@@ -5176,6 +5189,13 @@ static int rocksdb_init_func(void *const p) {
     DBUG_RETURN(1);
   }
 
+  if (rocksdb_ignore_datadic_errors)
+  {
+    sql_print_information(
+        "CAUTION: Running with rocksdb_ignore_datadic_errors=1. "
+        " This should only be used to perform repairs");
+  }
+
   if (rdb_check_rocksdb_corruption()) {
     // NO_LINT_DEBUG
     sql_print_error(
@@ -5607,7 +5627,14 @@ static int rocksdb_init_func(void *const p) {
   if (ddl_manager.init(&dict_manager, &cf_manager, rocksdb_validate_tables)) {
     // NO_LINT_DEBUG
     sql_print_error("RocksDB: Failed to initialize DDL manager.");
-    DBUG_RETURN(HA_EXIT_FAILURE);
+
+    if (rocksdb_ignore_datadic_errors)
+    {
+      sql_print_error("RocksDB: rocksdb_ignore_datadic_errors=1, "
+                      "trying to continue");
+    }
+    else
+      DBUG_RETURN(HA_EXIT_FAILURE);
   }
 
   Rdb_sst_info::init(rdb);
@@ -6674,9 +6701,18 @@ int ha_rocksdb::open(const char *const name, int mode, uint test_if_locked) {
                     "MyRocks has %u (%s hidden pk)",
                     table->s->keys, m_tbl_def->m_key_count,
                     has_hidden_pk(table)? "1" : "no");
-    my_error(ER_INTERNAL_ERROR, MYF(0),
-             "MyRocks: DDL mismatch. Check the error log for details");
-    DBUG_RETURN(HA_ERR_ROCKSDB_INVALID_TABLE);
+
+    if (rocksdb_ignore_datadic_errors)
+    {
+      sql_print_error("MyRocks: rocksdb_ignore_datadic_errors=1, "
+                      "trying to continue");
+    }
+    else
+    {
+      my_error(ER_INTERNAL_ERROR, MYF(0),
+               "MyRocks: DDL mismatch. Check the error log for details");
+      DBUG_RETURN(HA_ERR_ROCKSDB_INVALID_TABLE);
+    }
   }
 
 
@@ -11558,6 +11594,12 @@ void Rdb_drop_index_thread::run() {
               "from cf id %u. MyRocks data dictionary may "
               "get corrupted.",
               d.cf_id);
+          if (rocksdb_ignore_datadic_errors)
+          {
+            sql_print_error("RocksDB: rocksdb_ignore_datadic_errors=1, "
+                            "trying to continue");
+            continue;
+          }
           abort();
         }
         rocksdb::ColumnFamilyHandle *cfh = cf_manager.get_cf(d.cf_id);

storage/rocksdb/ha_rocksdb.h

@@ -1059,6 +1059,8 @@ const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_STABLE;
 
 extern bool prevent_myrocks_loading;
 
+extern uint32_t rocksdb_ignore_datadic_errors;
+
 void sql_print_verbose_info(const char *format, ...);
 
 }  // namespace myrocks

storage/rocksdb/mysql-test/rocksdb/r/rocksdb.result

@@ -932,6 +932,7 @@ rocksdb_force_flush_memtable_now	OFF
 rocksdb_force_index_records_in_range	0
 rocksdb_git_hash	#
 rocksdb_hash_index_allow_collision	ON
+rocksdb_ignore_datadic_errors	0
 rocksdb_ignore_unknown_options	ON
 rocksdb_index_type	kBinarySearch
 rocksdb_info_log_level	error_level

storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_ignore_datadic_errors_basic.result

@@ -0,0 +1,7 @@
+SET @start_global_value = @@global.ROCKSDB_IGNORE_DATADIC_ERRORS;
+SELECT @start_global_value;
+@start_global_value
+0
+"Trying to set variable @@global.ROCKSDB_IGNORE_DATADIC_ERRORS to 444. It should fail because it is readonly."
+SET @@global.ROCKSDB_IGNORE_DATADIC_ERRORS   = 444;
+ERROR HY000: Variable 'rocksdb_ignore_datadic_errors' is a read only variable

storage/rocksdb/mysql-test/rocksdb_sys_vars/t/rocksdb_ignore_datadic_errors_basic.test

@@ -0,0 +1,6 @@
+--source include/have_rocksdb.inc
+
+--let $sys_var=ROCKSDB_IGNORE_DATADIC_ERRORS
+--let $read_only=1
+--let $session=0
+--source include/rocksdb_sys_var.inc

storage/rocksdb/rdb_datadic.cc

@@ -5240,6 +5240,12 @@ void Rdb_dict_manager::log_start_drop_index(GL_INDEX_ID gl_index_id,
           "from index id (%u,%u). MyRocks data dictionary may "
           "get corrupted.",
           gl_index_id.cf_id, gl_index_id.index_id);
+      if (rocksdb_ignore_datadic_errors)
+      {
+        sql_print_error("RocksDB: rocksdb_ignore_datadic_errors=1, "
+                        "trying to continue");
+        return;
+      }
       abort();
     }
   }