Fix for the following bugs:

  - BUG#22306: STOP INSTANCE can not be applied for instances in Crashed,
    Failed and Abandoned;
  - BUG#23476: DROP INSTANCE does not work
  - BUG#23215: STOP INSTANCE takes too much time

BUG#22306:
The problem was that STOP INSTANCE checked that mysqld is up and running.
If it was not so, STOP INSTANCE reported an error. Now, STOP INSTANCE
reports an error if the instance has been started (mysqld can be down).

BUG#23476:
The problem was that DROP INSTANCE tried to stop inactive instance. The fix is
trivial.

BUG#23215:
The problem was that locks were not acquired properly, so the
instance-monitoring thread could not acquire the mutex, holded by the
query-processing thread.

The fix is to simplify locking scheme by moving instance-related information to
Instance-class out of Guardian-class. This allows to get rid of storing a
separate list of Instance-information in Guardian and keeping it synchronized
with the original list in Instance_map.


server-tools/instance-manager/commands.cc:
  1. Introduce Instance_cmd class -- base class for the commands
     that deal with the one instance;
  2. Remove Instance_map argument from command constructors;
  3. Ensure, that Instance Map and Instance are locked in the proper order;
  4. Polishing.
server-tools/instance-manager/commands.h:
  1. Introduce Instance_cmd class -- base class for the commands
     that deal with the one instance;
  2. Remove Instance_map argument from command constructors;
  3. Polishing.
server-tools/instance-manager/guardian.cc:
  1. Move "extended" instance information to the Instance-class.
     That allows to get rid of storing instance-related container and data in
     Guardian class, that significantly simplifies locking schema.
  2. Polishing.
server-tools/instance-manager/guardian.h:
  1. Move "extended" instance information to the Instance-class.
     That allows to get rid of storing instance-related container and data in
     Guardian class, that significantly simplifies locking schema.
  2. Polishing.
server-tools/instance-manager/instance.cc:
  1. Move "extended" instance information to the Instance-class.
  2. Introduce new state STOPPED to mark that guarded instance
     is stopped and should not be restarted by Guardian.
  3. Polishing.
server-tools/instance-manager/instance.h:
  1. Move "extended" instance information to the Instance-class.
  2. Introduce new state STOPPED to mark that guarded instance
     is stopped and should not be restarted by Guardian.
  3. Polishing.
server-tools/instance-manager/instance_map.cc:
  1. Move flush_instances() from Instance_map to Manager.
  2. Polishing.
server-tools/instance-manager/instance_map.h:
  1. Move flush_instances() from Instance_map to Manager.
  2. Polishing.
server-tools/instance-manager/instance_options.h:
  Polishing.
server-tools/instance-manager/manager.cc:
  1. Move flush_instances() from Instance_map to Manager.
  2. Polishing.
server-tools/instance-manager/manager.h:
  1. Move flush_instances() from Instance_map to Manager.
  2. Polishing.
server-tools/instance-manager/user_map.cc:
  Polishing.

server-tools/instance-manager/commands.h

@@ -30,7 +30,7 @@
 #endif
 
 
-/*
+/**
   Print all instances of this instance manager.
   Grammar: SHOW INSTANCES
 */
@@ -50,7 +50,7 @@ class Show_instances: public Command
 };
 
 
-/*
+/**
   Reread configuration file and refresh internal cache.
   Grammar: FLUSH INSTANCES
 */
@@ -66,11 +66,50 @@ class Flush_instances: public Command
 };
 
 
-/*
+/**
+  Base class for Instance-specific commands
+  (commands that operate on one instance).
+
+  Instance_cmd extends Command class by:
+    - an attribute for storing instance name;
+    - code to initialize instance name in constructor;
+    - an accessor to get instance name.
+*/
+
+class Instance_cmd : public Command
+{
+public:
+  Instance_cmd(const LEX_STRING *instance_name_arg);
+
+protected:
+  inline const LEX_STRING *get_instance_name() const
+  {
+    return instance_name.get_str();
+  }
+
+private:
+  Instance_name instance_name;
+};
+
+
+/**
   Abstract class for Instance-specific commands.
+
+  Abstract_instance_cmd extends Instance_cmd by providing a common
+  framework for writing command-implementations. Basically, the class
+  implements Command::execute() pure virtual function in the following
+  way:
+    - Lock Instance_map;
+    - Get an instance by name. Return an error, if there is no such
+      instance;
+    - Lock the instance;
+    - Unlock Instance_map;
+    - Call execute_impl(), which should be implemented in derived class;
+    - Unlock the instance;
+    - Send response to the client and return error status.
 */
 
-class Abstract_instance_cmd: public Command
+class Abstract_instance_cmd: public Instance_cmd
 {
 public:
   Abstract_instance_cmd(const LEX_STRING *instance_name_arg);
@@ -79,29 +118,24 @@ class Abstract_instance_cmd: public Command
   virtual int execute(st_net *net, ulong connection_id);
 
 protected:
-  /* MT-NOTE: this operation is called under acquired Instance_map's lock. */
+  /**
+    This operation is intended to contain command-specific implementation.
+
+    MT-NOTE: this operation is called under acquired Instance's lock.
+  */
   virtual int execute_impl(st_net *net, Instance *instance) = 0;
 
-  /*
+  /**
     This operation is invoked on successful return of execute_impl() and is
     intended to send closing data.
 
-    MT-NOTE: this operation is called under released Instance_map's lock.
+    MT-NOTE: this operation is called under released Instance's lock.
   */
   virtual int send_ok_response(st_net *net, ulong connection_id) = 0;
-
-protected:
-  inline const LEX_STRING *get_instance_name() const
-  {
-    return instance_name.get_str();
-  }
-
-private:
-  Instance_name instance_name;
 };
 
 
-/*
+/**
   Print status of an instance.
   Grammar: SHOW INSTANCE STATUS <instance_name>
 */
@@ -121,7 +155,7 @@ class Show_instance_status: public Abstract_instance_cmd
 };
 
 
-/*
+/**
   Print options of chosen instance.
   Grammar: SHOW INSTANCE OPTIONS <instance_name>
 */
@@ -141,7 +175,7 @@ class Show_instance_options: public Abstract_instance_cmd
 };
 
 
-/*
+/**
   Start an instance.
   Grammar: START INSTANCE <instance_name>
 */
@@ -157,7 +191,7 @@ class Start_instance: public Abstract_instance_cmd
 };
 
 
-/*
+/**
   Stop an instance.
   Grammar: STOP INSTANCE <instance_name>
 */
@@ -173,12 +207,12 @@ class Stop_instance: public Abstract_instance_cmd
 };
 
 
-/*
+/**
   Create an instance.
   Grammar: CREATE INSTANCE <instance_name> [<options>]
 */
 
-class Create_instance: public Command
+class Create_instance: public Instance_cmd
 {
 public:
   Create_instance(const LEX_STRING *instance_name_arg);
@@ -189,22 +223,15 @@ class Create_instance: public Command
 protected:
   virtual int execute(st_net *net, ulong connection_id);
 
-  inline const LEX_STRING *get_instance_name() const
-  {
-    return instance_name.get_str();
-  }
-
 private:
   bool parse_args(const char **text);
 
 private:
-  Instance_name instance_name;
-
   Named_value_arr options;
 };
 
 
-/*
+/**
   Drop an instance.
   Grammar: DROP INSTANCE <instance_name>
 
@@ -213,18 +240,17 @@ class Create_instance: public Command
   is removed from the instance map.
 */
 
-class Drop_instance: public Abstract_instance_cmd
+class Drop_instance: public Instance_cmd
 {
 public:
   Drop_instance(const LEX_STRING *instance_name_arg);
 
 protected:
-  virtual int execute_impl(st_net *net, Instance *instance);
-  virtual int send_ok_response(st_net *net, ulong connection_id);
+  virtual int execute(st_net *net, ulong connection_id);
 };
 
 
-/*
+/**
   Print requested part of the log.
   Grammar:
     SHOW <instance_name> LOG {ERROR | SLOW | GENERAL} size[, offset_from_end]
@@ -252,7 +278,7 @@ class Show_instance_log: public Abstract_instance_cmd
 };
 
 
-/*
+/**
   Shows the list of the log files, used by an instance.
   Grammar: SHOW <instance_name> LOG FILES
 */
@@ -272,7 +298,7 @@ class Show_instance_log_files: public Abstract_instance_cmd
 };
 
 
-/*
+/**
   Abstract class for option-management commands.
 */
 
@@ -312,7 +338,7 @@ class Abstract_option_cmd: public Command
 };
 
 
-/*
+/**
   Set an option for the instance.
   Grammar: SET instance_name.option[=option_value][, ...]
 */
@@ -329,7 +355,7 @@ class Set_option: public Abstract_option_cmd
 };
 
 
-/*
+/**
   Remove option of the instance.
   Grammar: UNSET instance_name.option[, ...]
 */
@@ -346,7 +372,7 @@ class Unset_option: public Abstract_option_cmd
 };
 
 
-/*
+/**
   Syntax error command.
 
   This command is issued if parser reported a syntax error. We need it to

server-tools/instance-manager/guardian.h

@@ -17,10 +17,12 @@
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */
 
 
-#include "thread_registry.h"
+#include <my_global.h>
 #include <my_sys.h>
 #include <my_list.h>
 
+#include "thread_registry.h"
+
 #if defined(__GNUC__) && defined(USE_PRAGMA_INTERFACE)
 #pragma interface
 #endif
@@ -28,7 +30,6 @@
 class Instance;
 class Instance_map;
 class Thread_registry;
-struct GUARD_NODE;
 
 /**
   The guardian thread is responsible for monitoring and restarting of guarded
@@ -38,97 +39,73 @@ struct GUARD_NODE;
 class Guardian: public Thread
 {
 public:
-  /* states of an instance */
-  enum enum_instance_state { NOT_STARTED= 1, STARTING, STARTED, JUST_CRASHED,
-                             CRASHED, CRASHED_AND_ABANDONED, STOPPING };
-
-  /*
-    The Guardian list node structure. Guardian utilizes it to store
-    guarded instances plus some additional info.
-  */
+  Guardian(Thread_registry *thread_registry_arg,
+           Instance_map *instance_map_arg);
+  ~Guardian();
 
-  struct GUARD_NODE
-  {
-    Instance *instance;
-    /* state of an instance (i.e. STARTED, CRASHED, etc.) */
-    enum_instance_state state;
-    /* the amount of attemts to restart instance (cleaned up at success) */
-    int restart_counter;
-    /* triggered at a crash */
-    time_t crash_moment;
-    /* General time field. Used to provide timeouts (at shutdown and restart) */
-    time_t last_checked;
-  };
-
-  /* Return client state name. */
-  static const char *get_instance_state_name(enum_instance_state state);
+  void init();
 
-  Guardian(Thread_registry *thread_registry_arg,
-           Instance_map *instance_map_arg,
-           uint monitoring_interval_arg);
-  virtual ~Guardian();
-  /* Initialize or refresh the list of guarded instances */
-  int init();
-  /* Request guardian shutdown. Stop instances if needed */
+public:
   void request_shutdown();
-  /* Start instance protection */
-  int guard(Instance *instance, bool nolock= FALSE);
-  /* Stop instance protection */
-  int stop_guard(Instance *instance);
-  /* Returns TRUE if guardian thread is stopped */
-  int is_stopped();
+
+  bool is_stopped();
+
   void lock();
   void unlock();
 
-  /*
-    Return an internal list node for the given instance if the instance is
-    managed by Guardian. Otherwise, return NULL.
+  void ping();
 
-    MT-NOTE: must be called under acquired lock.
-  */
-  LIST *find_instance_node(Instance *instance);
+protected:
+  virtual void run();
+
+private:
+  void stop_instances();
 
-  /* The operation is used to check if the instance is active or not. */
-  bool is_active(Instance *instance);
+  void process_instance(Instance *instance);
 
+private:
   /*
-    Return state of the given instance list node. The pointer must specify
-    a valid list node.
+    LOCK_guardian protectes the members in this section:
+      - shutdown_requested;
+      - stopped;
+
+    Also, it is used for COND_guardian.
   */
-  inline enum_instance_state get_instance_state(LIST *instance_node);
-protected:
-  /* Main funtion of the thread */
-  virtual void run();
+  pthread_mutex_t LOCK_guardian;
 
-public:
+  /*
+    Guardian's main loop waits on this condition. So, it should be signalled
+    each time, when instance state has been changed and we want Guardian to
+    wake up.
+
+    TODO: Change this to having data-scoped conditions, i.e. conditions,
+    which indicate that some data has been changed.
+  */
   pthread_cond_t COND_guardian;
 
-private:
-  /* Prepares Guardian shutdown. Stops instances is needed */
-  int stop_instances();
-  /* check instance state and act accordingly */
-  void process_instance(Instance *instance, GUARD_NODE *current_node,
-                        LIST **guarded_instances, LIST *elem);
+  /*
+    This variable is set to TRUE, when Manager thread is shutting down.
+    The flag is used by Guardian thread to understand that it's time to
+    finish.
+  */
+  bool shutdown_requested;
+
+  /*
+    This flag is set to TRUE on shutdown by Guardian thread, when all guarded
+    mysqlds are stopped.
 
-  int stopped;
+    The flag is used in the Manager thread to wait for Guardian to stop all
+    mysqlds.
+  */
+  bool stopped;
 
-private:
-  pthread_mutex_t LOCK_guardian;
   Thread_info thread_info;
-  int monitoring_interval;
   Thread_registry *thread_registry;
   Instance_map *instance_map;
-  LIST *guarded_instances;
-  MEM_ROOT alloc;
-  /* this variable is set to TRUE when we want to stop Guardian thread */
-  bool shutdown_requested;
-};
-
 
-inline Guardian::enum_instance_state
-Guardian::get_instance_state(LIST *instance_node)
-{
-  return ((GUARD_NODE *) instance_node->data)->state;
-}
+private:
+  Guardian(const Guardian &);
+  Guardian&operator =(const Guardian &);
+};
 
 #endif /* INCLUDES_MYSQL_INSTANCE_MANAGER_GUARDIAN_H */

server-tools/instance-manager/instance.h

@@ -30,7 +30,7 @@ class Instance_map;
 class Thread_registry;
 
 
-/*
+/**
   Instance_name -- the class represents instance name -- a string of length
   less than MAX_INSTANCE_NAME_SIZE.
 
@@ -68,72 +68,127 @@ class Instance_name
 class Instance
 {
 public:
-  /*
-    The following two constants defines name of the default mysqld-instance
-    ("mysqld").
+  /* States of an instance. */
+  enum enum_instance_state
+  {
+    STOPPED,
+    NOT_STARTED,
+    STARTING,
+    STARTED,
+    JUST_CRASHED,
+    CRASHED,
+    CRASHED_AND_ABANDONED,
+    STOPPING
+  };
+
+public:
+  /**
+    The constant defines name of the default mysqld-instance ("mysqld").
   */
   static const LEX_STRING DFLT_INSTANCE_NAME;
 
 public:
-  /*
-    The operation is intended to check whether string is a well-formed
-    instance name or not.
-  */
   static bool is_name_valid(const LEX_STRING *name);
-
-  /*
-    The operation is intended to check if the given instance name is
-    mysqld-compatible or not.
-  */
   static bool is_mysqld_compatible_name(const LEX_STRING *name);
 
 public:
   Instance();
-
   ~Instance();
+
   bool init(const LEX_STRING *name_arg);
   bool complete_initialization();
 
+public:
+  bool is_active();
+
   bool is_mysqld_running();
-  int start();
-  int stop();
-  /* send a signal to the instance */
+
+  bool start_mysqld();
+  bool stop_mysqld();
   void kill_mysqld(int signo);
-  bool is_crashed();
-  void set_crash_flag_n_wake_all();
 
-  /*
+  void lock();
+  void unlock();
+
+  const char *get_state_name();
+
+  void reset_stat();
+
+public:
+  /**
     The operation is intended to check if the instance is mysqld-compatible
     or not.
   */
   inline bool is_mysqld_compatible() const;
 
-  /*
+  /**
     The operation is intended to check if the instance is configured properly
     or not. Misconfigured instances are not managed.
   */
   inline bool is_configured() const;
 
+  /**
+    The operation returns TRUE if the instance is guarded and FALSE otherwise.
+  */
+  inline bool is_guarded() const;
+
+  /**
+    The operation returns name of the instance.
+  */
   inline const LEX_STRING *get_name() const;
 
+  /**
+    The operation returns the current state of the instance.
+
+    NOTE: At the moment should be used only for guarded instances.
+  */
+  inline enum_instance_state get_state() const;
+
+  /**
+    The operation changes the state of the instance.
+
+    NOTE: At the moment should be used only for guarded instances.
+    TODO: Make private.
+  */
+  inline void set_state(enum_instance_state new_state);
+
+  /**
+    The operation returns crashed flag.
+  */
+  inline bool is_crashed();
+
 public:
-  enum { DEFAULT_SHUTDOWN_DELAY= 35 };
+  /**
+    This attributes contains instance options.
+
+    TODO: Make private.
+  */
   Instance_options options;
 
 private:
-  /* This attributes is a flag, specifies if the instance has been crashed. */
+  /**
+    monitoring_thread_active is TRUE if there is a thread that monitors the
+    corresponding mysqld-process.
+  */
+  bool monitoring_thread_active;
+
+  /**
+    crashed is TRUE when corresponding mysqld-process has been died after
+    start.
+  */
   bool crashed;
 
-  /*
-    This attribute specifies if the instance is configured properly or not.
+  /**
+    configured is TRUE when the instance is configured and FALSE otherwise.
     Misconfigured instances are not managed.
   */
   bool configured;
 
   /*
-    This attribute specifies whether the instance is mysqld-compatible or not.
-    Mysqld-compatible instances can contain only mysqld-specific options.
-    At the moment an instance is mysqld-compatible if its name is "mysqld".
+    mysqld_compatible specifies whether the instance is mysqld-compatible
+    or not. Mysqld-compatible instances can contain only mysqld-specific
+    options. At the moment an instance is mysqld-compatible if its name is
+    "mysqld".
 
     The idea is that [mysqld] section should contain only mysqld-specific
     options (no Instance Manager-specific options) to be readable by mysqld
@@ -142,18 +197,36 @@ class Instance
   bool mysqld_compatible;
 
   /*
-    Mutex protecting the instance. Currently we use it to avoid the
-    double start of the instance. This happens when the instance is starting
-    and we issue the start command once more.
+    Mutex protecting the instance.
   */
   pthread_mutex_t LOCK_instance;
-  /*
-    This condition variable is used to wake threads waiting for instance to
-    stop in Instance::stop()
-  */
-  pthread_cond_t COND_instance_stopped;
 
+private:
+  /* Guarded-instance attributes. */
+
+  /* state of an instance (i.e. STARTED, CRASHED, etc.) */
+  enum_instance_state state;
+
+public:
+  /* the amount of attemts to restart instance (cleaned up at success) */
+  int restart_counter;
+
+  /* triggered at a crash */
+  time_t crash_moment;
+
+  /* General time field. Used to provide timeouts (at shutdown and restart) */
+  time_t last_checked;
+
+private:
+  static const char *get_instance_state_name(enum_instance_state state);
+
+private:
   void remove_pid();
+
+  bool wait_for_stop();
+
+private:
+  friend class Instance_monitor;
 };
 
 
@@ -169,9 +242,33 @@ inline bool Instance::is_configured() const
 }
 
 
+inline bool Instance::is_guarded() const
+{
+  return !options.nonguarded;
+}
+
+
 inline const LEX_STRING *Instance::get_name() const
 {
   return &options.instance_name;
 }
 
+
+inline Instance::enum_instance_state Instance::get_state() const
+{
+  return state;
+}
+
+
+inline void Instance::set_state(enum_instance_state new_state)
+{
+  state= new_state;
+}
+
+
+inline bool Instance::is_crashed()
+{
+  return crashed;
+}
+
 #endif /* INCLUDES_MYSQL_INSTANCE_MANAGER_INSTANCE_H */

server-tools/instance-manager/instance_map.cc

@@ -25,26 +25,18 @@
 #include <mysql_com.h>
 
 #include "buffer.h"
-#include "guardian.h"
 #include "instance.h"
 #include "log.h"
-#include "manager.h"
 #include "mysqld_error.h"
 #include "mysql_manager_error.h"
 #include "options.h"
 #include "priv.h"
 
-/*
-  Note:  As we are going to suppost different types of connections,
-  we shouldn't have connection-specific functions. To avoid it we could
-  put such functions to the Command-derived class instead.
-  The command could be easily constructed for a specific connection if
-  we would provide a special factory for each connection.
-*/
-
 C_MODE_START
 
-/* Procedure needed for HASH initialization */
+/**
+  HASH-routines: get key of instance for storing in hash.
+*/
 
 static byte* get_instance_key(const byte* u, uint* len,
                           my_bool __attribute__((unused)) t)
@@ -54,14 +46,18 @@ static byte* get_instance_key(const byte* u, uint* len,
   return (byte *) instance->options.instance_name.str;
 }
 
+/**
+  HASH-routines: cleanup handler.
+*/
+
 static void delete_instance(void *u)
 {
   Instance *instance= (Instance *) u;
   delete instance;
 }
 
-/*
-  The option handler to pass to the process_default_option_files finction.
+/**
+  The option handler to pass to the process_default_option_files function.
 
   SYNOPSIS
     process_option()
@@ -96,7 +92,7 @@ static int process_option(void *ctx, const char *group, const char *option)
 C_MODE_END
 
 
-/*
+/**
   Parse option string.
 
   SYNOPSIS
@@ -137,7 +133,7 @@ static void parse_option(const char *option_str,
 }
 
 
-/*
+/**
   Process one option from the configuration file.
 
   SYNOPSIS
@@ -151,6 +147,10 @@ static void parse_option(const char *option_str,
     process_option(). The caller ensures proper locking
     of the instance map object.
 */
+  /*
+    Process a given option and assign it to appropricate instance. This is
+    required for the option handler, passed to my_search_option_files().
+  */
 
 int Instance_map::process_one_option(const LEX_STRING *group,
                                      const char *option)
@@ -213,92 +213,97 @@ int Instance_map::process_one_option(const LEX_STRING *group,
 }
 
 
+/**
+  Instance_map constructor.
+*/
+
 Instance_map::Instance_map()
 {
   pthread_mutex_init(&LOCK_instance_map, 0);
 }
 
 
+/**
+  Initialize Instance_map internals.
+*/
+
 bool Instance_map::init()
 {
   return hash_init(&hash, default_charset_info, START_HASH_SIZE, 0, 0,
                    get_instance_key, delete_instance, 0);
 }
 
+
+/**
+  Reset Instance_map data.
+*/
+
+bool Instance_map::reset()
+{
+  hash_free(&hash);
+  return init();
+}
+
+
+/**
+  Instance_map destructor.
+*/
+
 Instance_map::~Instance_map()
 {
-  pthread_mutex_lock(&LOCK_instance_map);
+  lock();
+
+  /*
+    NOTE: it's necessary to synchronize on each instance before removal,
+    because Instance-monitoring thread can be still alive an hold the mutex
+    (because it is detached and we have no control over it).
+  */
+
+  while (true)
+  {
+    Iterator it(this);
+    Instance *instance= it.next();
+
+    if (!instance)
+      break;
+
+    instance->lock();
+    instance->unlock();
+
+    remove_instance(instance);
+  }
+
   hash_free(&hash);
-  pthread_mutex_unlock(&LOCK_instance_map);
+  unlock();
+
   pthread_mutex_destroy(&LOCK_instance_map);
 }
 
 
+/**
+  Lock Instance_map.
+*/
+
 void Instance_map::lock()
 {
   pthread_mutex_lock(&LOCK_instance_map);
 }
 
 
+/**
+  Unlock Instance_map.
+*/
+
 void Instance_map::unlock()
 {
   pthread_mutex_unlock(&LOCK_instance_map);
 }
 
-/*
-  Re-read instance configuration file.
-
-  SYNOPSIS
-    Instance_map::flush_instances()
 
-  DESCRIPTION
-    This function will:
-     - clear the current list of instances. This removes both
-       running and stopped instances.
-     - load a new instance configuration from the file.
-     - pass on the new map to the guardian thread: it will start
-       all instances that are marked `guarded' and not yet started.
-    Note, as the check whether an instance is started is currently
-    very simple (returns TRUE if there is a MySQL server running
-    at the given port), this function has some peculiar
-    side-effects:
-     * if the port number of a running instance was changed, the
-       old instance is forgotten, even if it was running. The new
-       instance will be started at the new port.
-     * if the configuration was changed in a way that two
-       instances swapped their port numbers, the guardian thread
-       will not notice that and simply report that both instances
-       are configured successfully and running.
-    In order to avoid such side effects one should never call
-    FLUSH INSTANCES without prior stop of all running instances.
-
-  NOTE: The operation should be invoked with the following locks acquired:
-    - Guardian;
-    - Instance_map;
+/**
+  Check if there is an active instance or not.
 */
 
-int Instance_map::flush_instances()
-{
-  int rc;
-
-  /*
-    Guardian thread relies on the instance map repository for guarding
-    instances. This is why refreshing instance map, we need (1) to stop
-    guardian (2) reload the instance map (3) reinitialize the guardian
-    with new instances.
-  */
-  hash_free(&hash);
-  hash_init(&hash, default_charset_info, START_HASH_SIZE, 0, 0,
-            get_instance_key, delete_instance, 0);
-
-  rc= load();
-  /* don't init guardian if we failed to load instances */
-  if (!rc)
-    guardian->init(); // TODO: check error status.
-  return rc;
-}
-
-
 bool Instance_map::is_there_active_instance()
 {
   Instance *instance;
@@ -306,29 +311,50 @@ bool Instance_map::is_there_active_instance()
 
   while ((instance= iterator.next()))
   {
-    if (guardian->find_instance_node(instance) != NULL ||
-        instance->is_mysqld_running())
-    {
+    bool active_instance_found;
+
+    instance->lock();
+    active_instance_found= instance->is_active();
+    instance->unlock();
+
+    if (active_instance_found)
       return TRUE;
   }
-  }
 
   return FALSE;
 }
 
 
+/**
+  Add an instance into the internal hash.
+
+  MT-NOTE: Instance Map must be locked before calling the operation.
+*/
+
 int Instance_map::add_instance(Instance *instance)
 {
   return my_hash_insert(&hash, (byte *) instance);
 }
 
 
+/**
+  Remove instance from the internal hash.
+
+  MT-NOTE: Instance Map must be locked before calling the operation.
+*/
+
 int Instance_map::remove_instance(Instance *instance)
 {
   return hash_delete(&hash, (byte *) instance);
 }
 
 
+/**
+  Create a new instance and register it in the internal hash.
+
+  MT-NOTE: Instance Map must be locked before calling the operation.
+*/
+
 int Instance_map::create_instance(const LEX_STRING *instance_name,
                                   const Named_value_arr *options)
 {
@@ -392,12 +418,22 @@ int Instance_map::create_instance(const LEX_STRING *instance_name,
 }
 
 
+/**
+  Return a pointer to the instance or NULL, if there is no such instance.
+
+  MT-NOTE: Instance Map must be locked before calling the operation.
+*/
+
 Instance * Instance_map::find(const LEX_STRING *name)
 {
   return (Instance *) hash_search(&hash, (byte *) name->str, name->length);
 }
 
 
+/**
+  Init instances command line arguments after all options have been loaded.
+*/
+
 bool Instance_map::complete_initialization()
 {
   bool mysqld_found;
@@ -455,7 +491,10 @@ bool Instance_map::complete_initialization()
 }
 
 
-/* load options from config files and create appropriate instance structures */
+/**
+  Load options from config files and create appropriate instance
+  structures.
+*/
 
 int Instance_map::load()
 {
@@ -505,8 +544,9 @@ int Instance_map::load()
 }
 
 
-/*--- Implementaton of the Instance map iterator class  ---*/
-
+/*************************************************************************
+  {{{ Instance_map::Iterator implementation.
+*************************************************************************/
 
 void Instance_map::Iterator::go_to_first()
 {
@@ -522,29 +562,12 @@ Instance *Instance_map::Iterator::next()
   return NULL;
 }
 
-
-const char *Instance_map::get_instance_state_name(Instance *instance)
-{
-  LIST *instance_node;
-
-  if (!instance->is_configured())
-    return "misconfigured";
-
-  if ((instance_node= guardian->find_instance_node(instance)) != NULL)
-  {
-    /* The instance is managed by Guardian: we can report precise state. */
-
-    return Guardian::get_instance_state_name(
-      guardian->get_instance_state(instance_node));
-  }
-
-  /* The instance is not managed by Guardian: we can report status only.  */
-
-  return instance->is_mysqld_running() ? "online" : "offline";
-}
+/*************************************************************************
+  }}}
+*************************************************************************/
 
 
-/*
+/**
   Create a new configuration section for mysqld-instance in the config file.
 
   SYNOPSIS

server-tools/instance-manager/instance_map.h

@@ -37,14 +37,17 @@ extern int create_instance_in_file(const LEX_STRING *instance_name,
                                    const Named_value_arr *options);
 
 
-/*
+/**
   Instance_map - stores all existing instances
 */
 
 class Instance_map
 {
 public:
-  /* Instance_map iterator */
+  /**
+    Instance_map iterator
+  */
+
   class Iterator
   {
   private:
@@ -58,79 +61,43 @@ class Instance_map
     void go_to_first();
     Instance *next();
   };
-  friend class Iterator;
+
 public:
-  /*
-    Return a pointer to the instance or NULL, if there is no such instance.
-    MT-NOTE: must be called under acquired lock.
-  */
   Instance *find(const LEX_STRING *name);
 
-  /* Clear the configuration cache and reload the configuration file. */
-  int flush_instances();
-
-  /* The operation is used to check if there is an active instance or not. */
   bool is_there_active_instance();
 
   void lock();
   void unlock();
 
   bool init();
+  bool reset();
 
-  /*
-    Process a given option and assign it to appropricate instance. This is
-    required for the option handler, passed to my_search_option_files().
-  */
-  int process_one_option(const LEX_STRING *group, const char *option);
+  int load();
 
-  /*
-    Add an instance into the internal hash.
+  int process_one_option(const LEX_STRING *group, const char *option);
 
-    MT-NOTE: the operation must be called under acquired lock.
-  */
   int add_instance(Instance *instance);
 
-  /*
-    Remove instance from the internal hash.
-
-    MT-NOTE: the operation must be called under acquired lock.
-  */
   int remove_instance(Instance *instance);
 
-  /*
-    Create a new instance and register it in the internal hash.
-
-    MT-NOTE: the operation must be called under acquired lock.
-  */
   int create_instance(const LEX_STRING *instance_name,
                       const Named_value_arr *options);
 
+public:
   Instance_map();
   ~Instance_map();
 
-  /*
-    Retrieve client state name of the given instance.
-
-    MT-NOTE: the options must be called under acquired locks of the following
-    objects:
-      - Instance_map;
-      - Guardian;
-  */
-  const char *get_instance_state_name(Instance *instance);
-
-public:
-  const char *mysqld_path;
-  Guardian *guardian;
-
 private:
-  /* loads options from config files */
-  int load();
-  /* inits instances argv's after all options have been loaded */
   bool complete_initialization();
+
 private:
   enum { START_HASH_SIZE = 16 };
   pthread_mutex_t LOCK_instance_map;
   HASH hash;
+
+private:
+  friend class Iterator;
 };
 
 #endif /* INCLUDES_MYSQL_INSTANCE_MANAGER_INSTANCE_MAP_H */

server-tools/instance-manager/instance_options.h

@@ -46,7 +46,6 @@ class Instance_options
   Instance_options();
   ~Instance_options();
 
-  /* fills in argv */
   bool complete_initialization();
 
   bool set_option(Named_value *option);

server-tools/instance-manager/manager.cc

@@ -37,6 +37,9 @@
 #include "user_map.h"
 
 
+/**********************************************************************
+ {{{ Platform-specific implementation.
+**********************************************************************/
 
 #ifndef __WIN__
 void set_signals(sigset_t *mask)
@@ -92,9 +95,13 @@ int my_sigwait(const sigset_t *set, int *sig)
 
 #endif
 
+/**********************************************************************
+  }}}
+**********************************************************************/
+
 
 /**********************************************************************
-  Implementation of checking the actual thread model.
+  {{{ Implementation of checking the actual thread model.
 ***********************************************************************/
 
 namespace { /* no-indent */
@@ -137,6 +144,10 @@ bool check_if_linux_threads(bool *linux_threads)
 
 }
 
+/**********************************************************************
+  }}}
+***********************************************************************/
+
 
 /**********************************************************************
   Manager implementation
@@ -152,23 +163,35 @@ bool Manager::linux_threads;
 #endif // __WIN__
 
 
+/**
+  Request shutdown of guardian and threads registered in Thread_registry.
+
+  SYNOPSIS
+    stop_all_threads()
+*/
+
 void Manager::stop_all_threads()
 {
   /*
-    Let guardian thread know that it should break it's processing cycle,
+    Let Guardian thread know that it should break it's processing cycle,
     once it wakes up.
   */
   p_guardian->request_shutdown();
-  /* wake guardian */
-  pthread_cond_signal(&p_guardian->COND_guardian);
-  /* stop all threads */
+
+  /* Stop all threads. */
   p_thread_registry->deliver_shutdown();
 }
 
 
-/*
-  manager - entry point to the main instance manager process: start
-  listener thread, write pid file and enter into signal handling.
+/**
+  Main manager function.
+
+  SYNOPSIS
+    main()
+
+  DESCRIPTION
+    This is an entry point to the main instance manager process:
+    start listener thread, write pid file and enter into signal handling.
     See also comments in mysqlmanager.cc to picture general Instance Manager
     architecture.
 
@@ -194,22 +217,33 @@ int Manager::main()
            (const char *) (linux_threads ? "LINUX threads" : "POSIX threads"));
 #endif // __WIN__
 
-  Thread_registry thread_registry;
   /*
-    All objects created in the manager() function live as long as
-    thread_registry lives, and thread_registry is alive until there are
-    working threads.
+    All objects created in the Manager object live as long as thread_registry
+    lives, and thread_registry is alive until there are working threads.
+
+    There are two main purposes of the Thread Registry:
+      1. Interrupt blocking I/O and signal condition variables in case of
+         shutdown;
+      2. Wait for detached threads before shutting down the main thread.
+
+    NOTE:
+      1. Handling shutdown can be done in more elegant manner by introducing
+         Event (or Condition) object with support of logical operations.
+      2. Using Thread Registry to wait for detached threads is definitely not
+         the best way, because when Thread Registry unregisters an thread, the
+         thread is still alive. Accurate way to wait for threads to stop is
+         not using detached threads and join all threads before shutdown.
   */
 
+  Thread_registry thread_registry;
   User_map user_map;
   Instance_map instance_map;
-  Guardian guardian(&thread_registry, &instance_map,
-                    Options::Main::monitoring_interval);
+  Guardian guardian(&thread_registry, &instance_map);
 
   Listener listener(&thread_registry, &user_map);
 
   p_instance_map= &instance_map;
-  p_guardian= instance_map.guardian= &guardian;
+  p_guardian= &guardian;
   p_thread_registry= &thread_registry;
   p_user_map= &user_map;
 
@@ -249,7 +283,7 @@ int Manager::main()
     }
   }
 
-  /* write Instance Manager pid file */
+  /* Write Instance Manager pid file. */
 
   log_info("IM pid file: '%s'; PID: %d.",
            (const char *) Options::Main::pid_file_name,
@@ -290,6 +324,7 @@ int Manager::main()
     permitted to process instances. And before flush_instances() has
     completed, there are no instances to guard.
   */
+
   if (guardian.start(Thread::DETACHED))
   {
     log_error("Can not start Guardian thread.");
@@ -298,22 +333,12 @@ int Manager::main()
 
   /* Load instances. */
 
-  {
-    instance_map.guardian->lock();
-    instance_map.lock();
-
-    int flush_instances_status= instance_map.flush_instances();
-
-    instance_map.unlock();
-    instance_map.guardian->unlock();
-
-    if (flush_instances_status)
+  if (Manager::flush_instances())
   {
     log_error("Can not init instances repository.");
     stop_all_threads();
     goto err;
   }
-  }
 
   /* Initialize the Listener. */
 
@@ -328,7 +353,8 @@ int Manager::main()
     After the list of guarded instances have been initialized,
     Guardian should start them.
   */
-  pthread_cond_signal(&guardian.COND_guardian);
+
+  guardian.ping();
 
   /* Main loop. */
 
@@ -381,7 +407,6 @@ int Manager::main()
       if (!guardian.is_stopped())
       {
         guardian.request_shutdown();
-        pthread_cond_signal(&guardian.COND_guardian);
       }
       else
       {
@@ -406,3 +431,64 @@ int Manager::main()
 #endif
   return rc;
 }
+
+
+/**
+  Re-read instance configuration file.
+
+  SYNOPSIS
+    flush_instances()
+
+  DESCRIPTION
+    This function will:
+     - clear the current list of instances. This removes both
+       running and stopped instances.
+     - load a new instance configuration from the file.
+     - pass on the new map to the guardian thread: it will start
+       all instances that are marked `guarded' and not yet started.
+
+    Note, as the check whether an instance is started is currently
+    very simple (returns TRUE if there is a MySQL server running
+    at the given port), this function has some peculiar
+    side-effects:
+     * if the port number of a running instance was changed, the
+       old instance is forgotten, even if it was running. The new
+       instance will be started at the new port.
+     * if the configuration was changed in a way that two
+       instances swapped their port numbers, the guardian thread
+       will not notice that and simply report that both instances
+       are configured successfully and running.
+
+    In order to avoid such side effects one should never call
+    FLUSH INSTANCES without prior stop of all running instances.
+*/
+
+bool Manager::flush_instances()
+{
+  p_instance_map->lock();
+
+  if (p_instance_map->is_there_active_instance())
+  {
+    p_instance_map->unlock();
+    return TRUE;
+  }
+
+  if (p_instance_map->reset())
+  {
+    p_instance_map->unlock();
+    return TRUE;
+  }
+
+  if (p_instance_map->load())
+  {
+    p_instance_map->unlock();
+    return TRUE; /* Don't init guardian if we failed to load instances. */
+  }
+
+  get_guardian()->init(); /* TODO: check error status. */
+  get_guardian()->ping();
+
+  p_instance_map->unlock();
+
+  return FALSE;
+}

server-tools/instance-manager/manager.h

@@ -19,6 +19,7 @@
 #if defined(__GNUC__) && defined(USE_PRAGMA_INTERFACE)
 #pragma interface
 #endif
+
 #include <my_global.h>
 
 class Guardian;
@@ -30,8 +31,12 @@ class Manager
 {
 public:
   static int main();
+
+  static bool flush_instances();
+
+public:
   /**
-    These methods return a non-zero value only for the duration
+    These methods return a non-NULL value only for the duration
     of main().
   */
   static Instance_map *get_instance_map() { return p_instance_map; }
@@ -39,6 +44,7 @@ class Manager
   static Thread_registry *get_thread_registry() { return p_thread_registry; }
   static User_map *get_user_map() { return p_user_map; }
 
+public:
 #ifndef __WIN__
   static bool is_linux_threads() { return linux_threads; }
 #endif // __WIN__

server-tools/instance-manager/user_map.cc

@@ -78,7 +78,7 @@ int User::init(const char *line)
               "User line: '%s'.",
               (int) password_length,
               (int) SCRAMBLED_PASSWORD_CHAR_LENGTH,
-             line);
+              (const char *) line);
     return 1;
   }