MDEV-31400 Simple plugin dependency resolution

We introduce simple plugin dependency. A plugin init function may
return HA_ERR_RETRY_INIT. If this happens during server startup when
the server is trying to initialise all plugins, the failed plugins
will be retried, until no more plugins succeed in initialisation or
want to be retried.

This will fix spider init bugs which is caused in part by its
dependency on Aria for initialisation.

The reason we need a new return code, instead of treating every
failure as a request for retry, is that it may be impossible to clean
up after a failed plugin initialisation. Take InnoDB for example, it
has a global variable `buf_page_cleaner_is_active`, which may not
satisfy an assertion during a second initialisation try, probably
because InnoDB does not expect the initialisation to be called
twice.

include/my_base.h

@@ -442,6 +442,7 @@ enum ha_base_keytype {
 #define HA_ERR_CRASHED		126	/* Indexfile is crashed */
 #define HA_ERR_WRONG_IN_RECORD	127	/* Record-file is crashed */
 #define HA_ERR_OUT_OF_MEM	128	/* Out of memory */
+#define HA_ERR_RETRY_INIT 129 /* Initialization failed and should be retried */
 #define HA_ERR_NOT_A_TABLE      130     /* not a MYI file - no signature */
 #define HA_ERR_WRONG_COMMAND	131	/* Command not supported */
 #define HA_ERR_OLD_FILE		132	/* old databasfile */

sql/handler.cc

@@ -550,6 +550,7 @@ int ha_initialize_handlerton(st_plugin_int *plugin)
 {
   handlerton *hton;
   static const char *no_exts[]= { 0 };
+  int ret= 0;
   DBUG_ENTER("ha_initialize_handlerton");
   DBUG_PRINT("plugin", ("initialize plugin: '%s'", plugin->name.str));
 
@@ -559,6 +560,7 @@ int ha_initialize_handlerton(st_plugin_int *plugin)
   {
     sql_print_error("Unable to allocate memory for plugin '%s' handlerton.",
                     plugin->name.str);
+    ret= 1;
     goto err_no_hton_memory;
   }
 
@@ -568,12 +570,16 @@ int ha_initialize_handlerton(st_plugin_int *plugin)
   hton->slot= HA_SLOT_UNDEF;
   /* Historical Requirement */
   plugin->data= hton; // shortcut for the future
-  if (plugin->plugin->init && plugin->plugin->init(hton))
-  {
-    sql_print_error("Plugin '%s' init function returned error.",
-		    plugin->name.str);
+  /* [remove after merge] notes on merge conflict (MDEV-31400):
+  10.5: 81cd93bbb81d75e7cf14dedede0c9ec0712ace68
+  10.6-10.11: 13ba00ff4933cfc1712676f323587504e453d1b5
+  11.0-11.2: 42f8be10f18163c4025710cf6a212e82bddb2f62
+  The 10.11->11.0 conflict is trivial, but the reference commit also
+  contains different non-conflict changes needs to be applied to 11.0
+  (and beyond).
+  */
+  if (plugin->plugin->init && (ret= plugin->plugin->init(hton)))
     goto err;
-  }
 
   // hton_ext_based_table_discovery() works only when discovery
   // is supported and the engine if file-based.
@@ -616,6 +622,7 @@ int ha_initialize_handlerton(st_plugin_int *plugin)
         if (idx == (int) DB_TYPE_DEFAULT)
         {
           sql_print_warning("Too many storage engines!");
+          ret= 1;
 	  goto err_deinit;
         }
         if (hton->db_type != DB_TYPE_UNKNOWN)
@@ -643,6 +650,7 @@ int ha_initialize_handlerton(st_plugin_int *plugin)
         {
           sql_print_error("Too many plugins loaded. Limit is %lu. "
                           "Failed on '%s'", (ulong) MAX_HA, plugin->name.str);
+          ret= 1;
           goto err_deinit;
         }
         hton->slot= total_ha++;
@@ -699,7 +707,7 @@ int ha_initialize_handlerton(st_plugin_int *plugin)
   resolve_sysvar_table_options(hton);
   update_discovery_counters(hton, 1);
 
-  DBUG_RETURN(0);
+  DBUG_RETURN(ret);
 
 err_deinit:
   /* 
@@ -717,7 +725,7 @@ int ha_initialize_handlerton(st_plugin_int *plugin)
   my_free(hton);
 err_no_hton_memory:
   plugin->data= NULL;
-  DBUG_RETURN(1);
+  DBUG_RETURN(ret);
 }
 
 int ha_init()

sql/sql_plugin.cc

@@ -1411,6 +1411,50 @@ void plugin_unlock_list(THD *thd, plugin_ref *list, uint count)
   DBUG_VOID_RETURN;
 }
 
+static void print_init_failed_error(st_plugin_int *p)
+{
+  sql_print_error("Plugin '%s' registration as a %s failed.",
+                  p->name.str,
+                  plugin_type_names[p->plugin->type].str);
+}
+
+static int plugin_do_initialize(struct st_plugin_int *plugin, uint &state)
+{
+  DBUG_ENTER("plugin_do_initialize");
+  mysql_mutex_assert_not_owner(&LOCK_plugin);
+  plugin_type_init init= plugin_type_initialize[plugin->plugin->type];
+  if (!init)
+    init= (plugin_type_init) plugin->plugin->init;
+  if (init)
+    if (int ret= init(plugin))
+    {
+      /* Plugin init failed and did not requested a retry */
+      if (ret != HA_ERR_RETRY_INIT)
+        print_init_failed_error(plugin);
+      DBUG_RETURN(ret);
+    }
+  state= PLUGIN_IS_READY; // plugin->init() succeeded
+
+  if (plugin->plugin->status_vars)
+  {
+    /*
+      historical ndb behavior caused MySQL plugins to specify
+      status var names in full, with the plugin name prefix.
+      this was never fixed in MySQL.
+      MariaDB fixes that but supports MySQL style too.
+    */
+    SHOW_VAR *show_vars= plugin->plugin->status_vars;
+    SHOW_VAR tmp_array[2]= {{plugin->plugin->name,
+                             (char *) plugin->plugin->status_vars, SHOW_ARRAY},
+                            {0, 0, SHOW_UNDEF}};
+    if (strncasecmp(show_vars->name, plugin->name.str, plugin->name.length))
+      show_vars= tmp_array;
+
+    if (add_status_vars(show_vars))
+      DBUG_RETURN(1);
+  }
+  DBUG_RETURN(0);
+}
 
 static int plugin_initialize(MEM_ROOT *tmp_root, struct st_plugin_int *plugin,
                              int *argc, char **argv, bool options_only)
@@ -1433,52 +1477,10 @@ static int plugin_initialize(MEM_ROOT *tmp_root, struct st_plugin_int *plugin,
   {
     ret= !options_only && plugin_is_forced(plugin);
     state= PLUGIN_IS_DISABLED;
-    goto err;
-  }
-
-  if (plugin_type_initialize[plugin->plugin->type])
-  {
-    if ((*plugin_type_initialize[plugin->plugin->type])(plugin))
-    {
-      sql_print_error("Plugin '%s' registration as a %s failed.",
-                      plugin->name.str, plugin_type_names[plugin->plugin->type].str);
-      goto err;
-    }
-  }
-  else if (plugin->plugin->init)
-  {
-    if (plugin->plugin->init(plugin))
-    {
-      sql_print_error("Plugin '%s' init function returned error.",
-                      plugin->name.str);
-      goto err;
-    }
   }
-  state= PLUGIN_IS_READY; // plugin->init() succeeded
-
-  if (plugin->plugin->status_vars)
-  {
-    /*
-      historical ndb behavior caused MySQL plugins to specify
-      status var names in full, with the plugin name prefix.
-      this was never fixed in MySQL.
-      MariaDB fixes that but supports MySQL style too.
-    */
-    SHOW_VAR *show_vars= plugin->plugin->status_vars;
-    SHOW_VAR tmp_array[2]= {
-      {plugin->plugin->name, (char*)plugin->plugin->status_vars, SHOW_ARRAY},
-      {0, 0, SHOW_UNDEF}
-    };
-    if (strncasecmp(show_vars->name, plugin->name.str, plugin->name.length))
-      show_vars= tmp_array;
-
-    if (add_status_vars(show_vars))
-      goto err;
-  }
-
-  ret= 0;
+  else
+    ret= plugin_do_initialize(plugin, state);
 
-err:
   if (ret)
     plugin_variables_deinit(plugin);
 
@@ -1557,7 +1559,7 @@ int plugin_init(int *argc, char **argv, int flags)
   uint i;
   struct st_maria_plugin **builtins;
   struct st_maria_plugin *plugin;
-  struct st_plugin_int tmp, *plugin_ptr, **reap;
+  struct st_plugin_int tmp, *plugin_ptr, **reap, **retry_end, **retry_start;
   MEM_ROOT tmp_root;
   bool reaped_mandatory_plugin= false;
   bool mandatory= true;
@@ -1707,11 +1709,16 @@ int plugin_init(int *argc, char **argv, int flags)
   */
 
   mysql_mutex_lock(&LOCK_plugin);
+  /* List of plugins to reap */
   reap= (st_plugin_int **) my_alloca((plugin_array.elements+1) * sizeof(void*));
   *(reap++)= NULL;
+  /* List of plugins to retry */
+  retry_start= retry_end=
+    (st_plugin_int **) my_alloca((plugin_array.elements+1) * sizeof(void*));
 
   for(;;)
   {
+    int error;
     for (i=0; i < MYSQL_MAX_PLUGIN_TYPE_NUM; i++)
     {
       HASH *hash= plugin_hash + plugin_type_initialization_order[i];
@@ -1720,15 +1727,51 @@ int plugin_init(int *argc, char **argv, int flags)
         plugin_ptr= (struct st_plugin_int *) my_hash_element(hash, idx);
         if (plugin_ptr->state == PLUGIN_IS_UNINITIALIZED)
         {
-          if (plugin_initialize(&tmp_root, plugin_ptr, argc, argv,
-                                (flags & PLUGIN_INIT_SKIP_INITIALIZATION)))
+          error= plugin_initialize(&tmp_root, plugin_ptr, argc, argv,
+                                   (flags & PLUGIN_INIT_SKIP_INITIALIZATION));
+          if (error)
           {
             plugin_ptr->state= PLUGIN_IS_DYING;
-            *(reap++)= plugin_ptr;
+            /* The plugin wants a retry of the initialisation,
+               possibly due to dependency on other plugins */
+            if (unlikely(error == HA_ERR_RETRY_INIT))
+              *(retry_end++)= plugin_ptr;
+            else
+              *(reap++)= plugin_ptr;
           }
         }
       }
     }
+    /* Retry plugins that asked for it */
+    while (retry_start < retry_end)
+    {
+      st_plugin_int **to_re_retry, **retrying;
+      for (to_re_retry= retrying= retry_start; retrying < retry_end; retrying++)
+      {
+        plugin_ptr= *retrying;
+        uint state= plugin_ptr->state;
+        mysql_mutex_unlock(&LOCK_plugin);
+        error= plugin_do_initialize(plugin_ptr, state);
+        mysql_mutex_lock(&LOCK_plugin);
+        plugin_ptr->state= state;
+        if (error == HA_ERR_RETRY_INIT)
+          *(to_re_retry++)= plugin_ptr;
+        else if (error)
+          *(reap++)= plugin_ptr;
+      }
+      /* If the retry list has not changed, i.e. if all retry attempts
+      result in another retry request, empty the retry list */
+      if (to_re_retry == retry_end)
+        while (to_re_retry > retry_start)
+        {
+          plugin_ptr= *(--to_re_retry);
+          *(reap++)= plugin_ptr;
+          /** `plugin_do_initialize()' did not print any error in this
+          case, so we do it here. */
+          print_init_failed_error(plugin_ptr);
+        }
+      retry_end= to_re_retry;
+    }
 
     /* load and init plugins from the plugin table (unless done already) */
     if (flags & PLUGIN_INIT_SKIP_PLUGIN_TABLE)
@@ -1754,6 +1797,7 @@ int plugin_init(int *argc, char **argv, int flags)
   }
 
   mysql_mutex_unlock(&LOCK_plugin);
+  my_afree(retry_start);
   my_afree(reap);
   if (reaped_mandatory_plugin)
     goto err;