ndb -

  refactor testSRBank and create new class NdbMixRestarter
parent ee944816
/* Copyright (C) 2003 MySQL AB
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */
#ifndef NDBT_MIX_RESTARTER_HPP
#define NDBT_MIX_RESTARTER_HPP
#include <mgmapi.h>
#include <Vector.hpp>
#include <BaseString.hpp>
#include "NdbRestarter.hpp"
#include "NDBT_Test.hpp"
#define NMR_SR "SR"
#define NMR_SR_THREADS "SR_ThreadCount"
#define NMR_SR_THREADS_STOPPED "SR_ThreadsStoppedCount"
#define NMR_SR_VALIDATE_THREADS "SR_ValidateThreadCount"
#define NMR_SR_VALIDATE_THREADS_DONE "SR_ValidateThreadsDoneCount"
class NdbMixRestarter : public NdbRestarter
{
public:
enum RestartTypeMask
{
RTM_RestartCluster = 0x01,
RTM_RestartNode = 0x02,
RTM_RestartNodeInitial = 0x04,
RTM_StopNode = 0x08,
RTM_StopNodeInitial = 0x10,
RTM_StartNode = 0x20,
RTM_COUNT = 6,
RTM_ALL = 0xFF,
RTM_SR = RTM_RestartCluster,
RTM_NR = 0x2 | 0x4 | 0x8 | 0x10 | 0x20
};
enum SR_State {
SR_RUNNING = 0,
SR_STOPPING = 1,
SR_STOPPED = 2,
SR_VALIDATING = 3
};
NdbMixRestarter(const char* _addr = 0);
~NdbMixRestarter();
void setRestartTypeMask(Uint32 mask);
int runUntilStopped(NDBT_Context* ctx, NDBT_Step* step, Uint32 freq);
int runPeriod(NDBT_Context* ctx, NDBT_Step* step, Uint32 time, Uint32 freq);
int init(NDBT_Context* ctx, NDBT_Step* step);
int dostep(NDBT_Context* ctx, NDBT_Step* step);
int finish(NDBT_Context* ctx, NDBT_Step* step);
private:
Uint32 m_mask;
Vector<ndb_mgm_node_state> m_nodes;
int restart_cluster(NDBT_Context* ctx, NDBT_Step* step, bool abort = true);
};
#endif
...@@ -20,9 +20,16 @@ ...@@ -20,9 +20,16 @@
#include <NdbBackup.hpp> #include <NdbBackup.hpp>
#include "bank/Bank.hpp" #include "bank/Bank.hpp"
#include <NdbMixRestarter.hpp>
bool disk = false; bool disk = false;
#define CHECK(b) if (!(b)) { \
g_err << "ERR: "<< step->getName() \
<< " failed on line " << __LINE__ << endl; \
result = NDBT_FAILED; \
continue; }
int runCreateBank(NDBT_Context* ctx, NDBT_Step* step){ int runCreateBank(NDBT_Context* ctx, NDBT_Step* step){
Bank bank(ctx->m_cluster_connection); Bank bank(ctx->m_cluster_connection);
int overWriteExisting = true; int overWriteExisting = true;
...@@ -37,21 +44,25 @@ int runCreateBank(NDBT_Context* ctx, NDBT_Step* step){ ...@@ -37,21 +44,25 @@ int runCreateBank(NDBT_Context* ctx, NDBT_Step* step){
* SR 1 - shutdown in progress * SR 1 - shutdown in progress
* SR 2 - restart in progress * SR 2 - restart in progress
*/ */
int runBankTimer(NDBT_Context* ctx, NDBT_Step* step){ int
runBankTimer(NDBT_Context* ctx, NDBT_Step* step){
int wait = 5; // Max seconds between each "day" int wait = 5; // Max seconds between each "day"
int yield = 1; // Loops before bank returns int yield = 1; // Loops before bank returns
ctx->incProperty("ThreadCount"); ctx->incProperty(NMR_SR_THREADS);
while (!ctx->isTestStopped()) while (!ctx->isTestStopped())
{ {
Bank bank(ctx->m_cluster_connection); Bank bank(ctx->m_cluster_connection);
while(!ctx->isTestStopped() && ctx->getProperty("SR") <= 1) while(!ctx->isTestStopped() &&
ctx->getProperty(NMR_SR) <= NdbMixRestarter::SR_STOPPING)
{
if(bank.performIncreaseTime(wait, yield) == NDBT_FAILED) if(bank.performIncreaseTime(wait, yield) == NDBT_FAILED)
break; break;
}
ndbout_c("runBankTimer is stopped"); ndbout_c("runBankTimer is stopped");
ctx->incProperty("ThreadStopped"); ctx->incProperty(NMR_SR_THREADS_STOPPED);
if(ctx->getPropertyWait("SR", (Uint32)0)) if(ctx->getPropertyWait(NMR_SR, NdbMixRestarter::SR_RUNNING))
break; break;
} }
return NDBT_OK; return NDBT_OK;
...@@ -61,17 +72,18 @@ int runBankTransactions(NDBT_Context* ctx, NDBT_Step* step){ ...@@ -61,17 +72,18 @@ int runBankTransactions(NDBT_Context* ctx, NDBT_Step* step){
int wait = 0; // Max ms between each transaction int wait = 0; // Max ms between each transaction
int yield = 1; // Loops before bank returns int yield = 1; // Loops before bank returns
ctx->incProperty("ThreadCount"); ctx->incProperty(NMR_SR_THREADS);
while (!ctx->isTestStopped()) while (!ctx->isTestStopped())
{ {
Bank bank(ctx->m_cluster_connection); Bank bank(ctx->m_cluster_connection);
while(!ctx->isTestStopped() && ctx->getProperty("SR") <= 1) while(!ctx->isTestStopped() &&
ctx->getProperty(NMR_SR) <= NdbMixRestarter::SR_STOPPING)
if(bank.performTransactions(0, 1) == NDBT_FAILED) if(bank.performTransactions(0, 1) == NDBT_FAILED)
break; break;
ndbout_c("runBankTransactions is stopped"); ndbout_c("runBankTransactions is stopped");
ctx->incProperty("ThreadStopped"); ctx->incProperty(NMR_SR_THREADS_STOPPED);
if(ctx->getPropertyWait("SR", (Uint32)0)) if(ctx->getPropertyWait(NMR_SR, NdbMixRestarter::SR_RUNNING))
break; break;
} }
return NDBT_OK; return NDBT_OK;
...@@ -81,269 +93,98 @@ int runBankGL(NDBT_Context* ctx, NDBT_Step* step){ ...@@ -81,269 +93,98 @@ int runBankGL(NDBT_Context* ctx, NDBT_Step* step){
int yield = 1; // Loops before bank returns int yield = 1; // Loops before bank returns
int result = NDBT_OK; int result = NDBT_OK;
ctx->incProperty("ThreadCount"); ctx->incProperty(NMR_SR_THREADS);
while (ctx->isTestStopped() == false) while (ctx->isTestStopped() == false)
{ {
Bank bank(ctx->m_cluster_connection); Bank bank(ctx->m_cluster_connection);
while(!ctx->isTestStopped() && ctx->getProperty("SR") <= 1) while(!ctx->isTestStopped() &&
ctx->getProperty(NMR_SR) <= NdbMixRestarter::SR_STOPPING)
if (bank.performMakeGLs(yield) != NDBT_OK) if (bank.performMakeGLs(yield) != NDBT_OK)
{ {
if(ctx->getProperty("SR") != 0) if(ctx->getProperty(NMR_SR) != NdbMixRestarter::SR_RUNNING)
break; break;
ndbout << "bank.performMakeGLs FAILED" << endl; ndbout << "bank.performMakeGLs FAILED" << endl;
abort();
return NDBT_FAILED; return NDBT_FAILED;
} }
ndbout_c("runBankGL is stopped"); ndbout_c("runBankGL is stopped");
ctx->incProperty("ThreadStopped"); ctx->incProperty(NMR_SR_THREADS_STOPPED);
if(ctx->getPropertyWait("SR", (Uint32)0)) if(ctx->getPropertyWait(NMR_SR, NdbMixRestarter::SR_RUNNING))
break; break;
} }
return NDBT_OK; return NDBT_OK;
} }
int runBankSum(NDBT_Context* ctx, NDBT_Step* step){
Bank bank(ctx->m_cluster_connection);
int wait = 2000; // Max ms between each sum of accounts
int yield = 1; // Loops before bank returns
int result = NDBT_OK;
while (ctx->isTestStopped() == false) {
if (bank.performSumAccounts(wait, yield) != NDBT_OK){
ndbout << "bank.performSumAccounts FAILED" << endl;
result = NDBT_FAILED;
}
}
return result ;
}
#define CHECK(b) if (!(b)) { \
g_err << "ERR: "<< step->getName() \
<< " failed on line " << __LINE__ << endl; \
result = NDBT_FAILED; \
continue; }
static
int int
restart_cluster(NDBT_Context* ctx, NDBT_Step* step, NdbRestarter& restarter) runBankSrValidator(NDBT_Context* ctx, NDBT_Step* step)
{ {
bool abort = true;
int timeout = 180;
int result = NDBT_OK;
do ctx->incProperty(NMR_SR_VALIDATE_THREADS);
{
ndbout << " -- Shutting down " << endl;
ctx->setProperty("SR", 1);
CHECK(restarter.restartAll(false, true, abort) == 0);
ctx->setProperty("SR", 2);
CHECK(restarter.waitClusterNoStart(timeout) == 0);
Uint32 cnt = ctx->getProperty("ThreadCount");
Uint32 curr= ctx->getProperty("ThreadStopped");
while(curr != cnt && !ctx->isTestStopped())
{
ndbout_c("%d %d", curr, cnt);
NdbSleep_MilliSleep(100);
curr= ctx->getProperty("ThreadStopped");
}
ctx->setProperty("ThreadStopped", (Uint32)0); while(!ctx->isTestStopped())
CHECK(restarter.startAll() == 0);
CHECK(restarter.waitClusterStarted(timeout) == 0);
ndbout << " -- Validating starts " << endl;
{ {
if (ctx->getPropertyWait(NMR_SR, NdbMixRestarter::SR_VALIDATING))
break;
int wait = 0; int wait = 0;
int yield = 1; int yield = 1;
Bank bank(ctx->m_cluster_connection); Bank bank(ctx->m_cluster_connection);
if (bank.performSumAccounts(wait, yield) != 0) if (bank.performSumAccounts(wait, yield) != 0)
{ {
ndbout << "bank.performSumAccounts FAILED" << endl; ndbout << "bank.performSumAccounts FAILED" << endl;
abort();
return NDBT_FAILED; return NDBT_FAILED;
} }
if (bank.performValidateAllGLs() != 0) if (bank.performValidateAllGLs() != 0)
{ {
ndbout << "bank.performValidateAllGLs FAILED" << endl; ndbout << "bank.performValidateAllGLs FAILED" << endl;
abort();
return NDBT_FAILED; return NDBT_FAILED;
} }
}
ndbout << " -- Validating complete " << endl; ctx->incProperty(NMR_SR_VALIDATE_THREADS_DONE);
} while(0);
ctx->setProperty("SR", (Uint32)0);
ctx->broadcast();
return result;
}
static if (ctx->getPropertyWait(NMR_SR, NdbMixRestarter::SR_RUNNING))
ndb_mgm_node_state*
select_node_to_stop(Vector<ndb_mgm_node_state>& nodes)
{
Uint32 i, j;
Vector<ndb_mgm_node_state*> alive_nodes;
for(i = 0; i<nodes.size(); i++)
{
ndb_mgm_node_state* node = &nodes[i];
if (node->node_status == NDB_MGM_NODE_STATUS_STARTED)
alive_nodes.push_back(node);
}
Vector<ndb_mgm_node_state*> victims;
// Remove those with one in node group
for(i = 0; i<alive_nodes.size(); i++)
{
int group = alive_nodes[i]->node_group;
for(j = 0; j<alive_nodes.size(); j++)
{
if (i != j && alive_nodes[j]->node_group == group)
{
victims.push_back(alive_nodes[i]);
break; break;
} }
}
}
if (victims.size()) return NDBT_OK;
{
int victim = rand() % victims.size();
return victims[victim];
}
else
{
return 0;
}
} }
static #if 0
ndb_mgm_node_state* int runBankSum(NDBT_Context* ctx, NDBT_Step* step){
select_node_to_start(Vector<ndb_mgm_node_state>& nodes) Bank bank(ctx->m_cluster_connection);
{ int wait = 2000; // Max ms between each sum of accounts
Uint32 i, j; int yield = 1; // Loops before bank returns
Vector<ndb_mgm_node_state*> victims; int result = NDBT_OK;
for(i = 0; i<nodes.size(); i++)
{
ndb_mgm_node_state* node = &nodes[i];
if (node->node_status == NDB_MGM_NODE_STATUS_NOT_STARTED)
victims.push_back(node);
}
if (victims.size()) while (ctx->isTestStopped() == false)
{ {
int victim = rand() % victims.size(); if (bank.performSumAccounts(wait, yield) != NDBT_OK){
return victims[victim]; ndbout << "bank.performSumAccounts FAILED" << endl;
result = NDBT_FAILED;
} }
else
{
return 0;
} }
return result ;
} }
#endif
enum Action {
AA_RestartCluster = 0x1,
AA_RestartNode = 0x2,
AA_StopNode = 0x4,
AA_StartNode = 0x8,
AA_COUNT = 4
};
int int
runMixRestart(NDBT_Context* ctx, NDBT_Step* step) runMixRestart(NDBT_Context* ctx, NDBT_Step* step)
{ {
int result = NDBT_OK; int result = NDBT_OK;
NdbMixRestarter res;
int runtime = ctx->getNumLoops(); int runtime = ctx->getNumLoops();
int sleeptime = ctx->getNumRecords(); int sleeptime = ctx->getNumRecords();
NdbRestarter restarter; Uint32 mask = ctx->getProperty("Type", ~(Uint32)0);
int timeout = 180; res.setRestartTypeMask(mask);
Uint32 type = ctx->getProperty("Type", ~(Uint32)0);
restarter.waitClusterStarted();
Vector<ndb_mgm_node_state> nodes;
nodes = restarter.ndbNodes;
#if 0
for (Uint32 i = 0; i<restarter.ndbNodes.size(); i++)
nodes.push_back(restarter.ndbNodes[i]);
#endif
Uint32 now;
const Uint32 stop = time(0)+ runtime;
while(!ctx->isTestStopped() && ((now= time(0)) < stop) && result == NDBT_OK)
{
ndbout << " -- Sleep " << sleeptime << "s " << endl;
int cnt = sleeptime;
while (cnt-- && !ctx->isTestStopped())
NdbSleep_SecSleep(1);
if (ctx->isTestStopped())
return NDBT_FAILED;
ndb_mgm_node_state* node = 0;
int action;
loop:
while(((action = (1 << (rand() % AA_COUNT))) & type) == 0);
switch(action){
case AA_RestartCluster:
if (restart_cluster(ctx, step, restarter))
return NDBT_FAILED;
for (Uint32 i = 0; i<nodes.size(); i++)
nodes[i].node_status = NDB_MGM_NODE_STATUS_STARTED;
break;
case AA_RestartNode:
case AA_StopNode:
{
if ((node = select_node_to_stop(nodes)) == 0)
goto loop;
if (action == AA_RestartNode)
g_err << "Restarting " << node->node_id << endl;
else
g_err << "Stopping " << node->node_id << endl;
if (restarter.restartOneDbNode(node->node_id, false, true, true)) if (res.runPeriod(ctx, step, runtime, sleeptime))
return NDBT_FAILED;
if (restarter.waitNodesNoStart(&node->node_id, 1))
return NDBT_FAILED;
node->node_status = NDB_MGM_NODE_STATUS_NOT_STARTED;
if (action == AA_StopNode)
break;
else
goto start;
}
case AA_StartNode:
if ((node = select_node_to_start(nodes)) == 0)
goto loop;
start:
g_err << "Starting " << node->node_id << endl;
if (restarter.startNodes(&node->node_id, 1))
return NDBT_FAILED;
if (restarter.waitNodesStarted(&node->node_id, 1))
return NDBT_FAILED;
node->node_status = NDB_MGM_NODE_STATUS_STARTED;
break;
}
}
Vector<int> not_started;
{
ndb_mgm_node_state* node = 0;
while((node = select_node_to_start(nodes)))
{ {
not_started.push_back(node->node_id); abort();
node->node_status = NDB_MGM_NODE_STATUS_STARTED;
}
}
if (not_started.size())
{
g_err << "Starting stopped nodes " << endl;
if (restarter.startNodes(not_started.getBase(), not_started.size()))
return NDBT_FAILED;
if (restarter.waitClusterStarted())
return NDBT_FAILED; return NDBT_FAILED;
} }
...@@ -351,7 +192,8 @@ runMixRestart(NDBT_Context* ctx, NDBT_Step* step) ...@@ -351,7 +192,8 @@ runMixRestart(NDBT_Context* ctx, NDBT_Step* step)
return NDBT_OK; return NDBT_OK;
} }
int runDropBank(NDBT_Context* ctx, NDBT_Step* step){ int
runDropBank(NDBT_Context* ctx, NDBT_Step* step){
Bank bank(ctx->m_cluster_connection); Bank bank(ctx->m_cluster_connection);
if (bank.dropBank() != NDBT_OK) if (bank.dropBank() != NDBT_OK)
return NDBT_FAILED; return NDBT_FAILED;
...@@ -367,7 +209,7 @@ TESTCASE("SR", ...@@ -367,7 +209,7 @@ TESTCASE("SR",
"3. Restart ndb and verify consistency\n" "3. Restart ndb and verify consistency\n"
"4. Drop bank\n") "4. Drop bank\n")
{ {
TC_PROPERTY("Type", AA_RestartCluster); TC_PROPERTY("Type", NdbMixRestarter::RTM_SR);
INITIALIZER(runCreateBank); INITIALIZER(runCreateBank);
STEP(runBankTimer); STEP(runBankTimer);
STEP(runBankTransactions); STEP(runBankTransactions);
...@@ -381,6 +223,7 @@ TESTCASE("SR", ...@@ -381,6 +223,7 @@ TESTCASE("SR",
STEP(runBankTransactions); STEP(runBankTransactions);
STEP(runBankTransactions); STEP(runBankTransactions);
STEP(runBankGL); STEP(runBankGL);
STEP(runBankSrValidator);
STEP(runMixRestart); STEP(runMixRestart);
} }
TESTCASE("NR", TESTCASE("NR",
...@@ -390,7 +233,7 @@ TESTCASE("NR", ...@@ -390,7 +233,7 @@ TESTCASE("NR",
"3. Restart ndb and verify consistency\n" "3. Restart ndb and verify consistency\n"
"4. Drop bank\n") "4. Drop bank\n")
{ {
TC_PROPERTY("Type", AA_RestartNode | AA_StopNode | AA_StartNode); TC_PROPERTY("Type", NdbMixRestarter::RTM_NR);
INITIALIZER(runCreateBank); INITIALIZER(runCreateBank);
STEP(runBankTimer); STEP(runBankTimer);
STEP(runBankTransactions); STEP(runBankTransactions);
...@@ -414,7 +257,7 @@ TESTCASE("Mix", ...@@ -414,7 +257,7 @@ TESTCASE("Mix",
"3. Restart ndb and verify consistency\n" "3. Restart ndb and verify consistency\n"
"4. Drop bank\n") "4. Drop bank\n")
{ {
TC_PROPERTY("Type", ~0); TC_PROPERTY("Type", NdbMixRestarter::RTM_ALL);
INITIALIZER(runCreateBank); INITIALIZER(runCreateBank);
STEP(runBankTimer); STEP(runBankTimer);
STEP(runBankTransactions); STEP(runBankTransactions);
...@@ -429,6 +272,7 @@ TESTCASE("Mix", ...@@ -429,6 +272,7 @@ TESTCASE("Mix",
STEP(runBankTransactions); STEP(runBankTransactions);
STEP(runBankGL); STEP(runBankGL);
STEP(runMixRestart); STEP(runMixRestart);
STEP(runBankSrValidator);
FINALIZER(runDropBank); FINALIZER(runDropBank);
} }
NDBT_TESTSUITE_END(testSRBank); NDBT_TESTSUITE_END(testSRBank);
......
...@@ -24,7 +24,7 @@ libNDBT_a_SOURCES = \ ...@@ -24,7 +24,7 @@ libNDBT_a_SOURCES = \
NdbRestarter.cpp NdbRestarts.cpp NDBT_Output.cpp \ NdbRestarter.cpp NdbRestarts.cpp NDBT_Output.cpp \
NdbBackup.cpp NdbConfig.cpp NdbGrep.cpp NDBT_Table.cpp \ NdbBackup.cpp NdbConfig.cpp NdbGrep.cpp NDBT_Table.cpp \
NdbSchemaCon.cpp NdbSchemaOp.cpp getarg.c \ NdbSchemaCon.cpp NdbSchemaOp.cpp getarg.c \
CpcClient.cpp CpcClient.cpp NdbMixRestarter.cpp
INCLUDES_LOC = -I$(top_srcdir)/storage/ndb/src/common/mgmcommon -I$(top_srcdir)/storage/ndb/include/mgmcommon -I$(top_srcdir)/storage/ndb/include/kernel -I$(top_srcdir)/storage/ndb/src/mgmapi INCLUDES_LOC = -I$(top_srcdir)/storage/ndb/src/common/mgmcommon -I$(top_srcdir)/storage/ndb/include/mgmcommon -I$(top_srcdir)/storage/ndb/include/kernel -I$(top_srcdir)/storage/ndb/src/mgmapi
......
...@@ -146,8 +146,10 @@ void NDBT_Context::setProperty(const char* _name, Uint32 _val){ ...@@ -146,8 +146,10 @@ void NDBT_Context::setProperty(const char* _name, Uint32 _val){
NdbMutex_Lock(propertyMutexPtr); NdbMutex_Lock(propertyMutexPtr);
const bool b = props.put(_name, _val, true); const bool b = props.put(_name, _val, true);
assert(b == true); assert(b == true);
NdbCondition_Broadcast(propertyCondPtr);
NdbMutex_Unlock(propertyMutexPtr); NdbMutex_Unlock(propertyMutexPtr);
} }
void void
NDBT_Context::decProperty(const char * name){ NDBT_Context::decProperty(const char * name){
NdbMutex_Lock(propertyMutexPtr); NdbMutex_Lock(propertyMutexPtr);
...@@ -159,6 +161,7 @@ NDBT_Context::decProperty(const char * name){ ...@@ -159,6 +161,7 @@ NDBT_Context::decProperty(const char * name){
NdbCondition_Broadcast(propertyCondPtr); NdbCondition_Broadcast(propertyCondPtr);
NdbMutex_Unlock(propertyMutexPtr); NdbMutex_Unlock(propertyMutexPtr);
} }
void void
NDBT_Context::incProperty(const char * name){ NDBT_Context::incProperty(const char * name){
NdbMutex_Lock(propertyMutexPtr); NdbMutex_Lock(propertyMutexPtr);
...@@ -173,6 +176,7 @@ void NDBT_Context::setProperty(const char* _name, const char* _val){ ...@@ -173,6 +176,7 @@ void NDBT_Context::setProperty(const char* _name, const char* _val){
NdbMutex_Lock(propertyMutexPtr); NdbMutex_Lock(propertyMutexPtr);
const bool b = props.put(_name, _val); const bool b = props.put(_name, _val);
assert(b == true); assert(b == true);
NdbCondition_Broadcast(propertyCondPtr);
NdbMutex_Unlock(propertyMutexPtr); NdbMutex_Unlock(propertyMutexPtr);
} }
......
/* Copyright (C) 2003 MySQL AB
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */
#include "NdbMixRestarter.hpp"
NdbMixRestarter::NdbMixRestarter(const char* _addr) :
NdbRestarter(_addr),
m_mask(~(Uint32)0)
{
}
NdbMixRestarter::~NdbMixRestarter()
{
}
#define CHECK(b) if (!(b)) { \
ndbout << "ERR: "<< step->getName() \
<< " failed on line " << __LINE__ << endl; \
result = NDBT_FAILED; \
continue; }
int
NdbMixRestarter::restart_cluster(NDBT_Context* ctx,
NDBT_Step* step,
bool stopabort)
{
int timeout = 180;
int result = NDBT_OK;
do
{
ctx->setProperty(NMR_SR_THREADS_STOPPED, (Uint32)0);
ctx->setProperty(NMR_SR_VALIDATE_THREADS_DONE, (Uint32)0);
ndbout << " -- Shutting down " << endl;
ctx->setProperty(NMR_SR, NdbMixRestarter::SR_STOPPING);
CHECK(restartAll(false, true, stopabort) == 0);
ctx->setProperty(NMR_SR, NdbMixRestarter::SR_STOPPED);
CHECK(waitClusterNoStart(timeout) == 0);
Uint32 cnt = ctx->getProperty(NMR_SR_THREADS);
Uint32 curr= ctx->getProperty(NMR_SR_THREADS_STOPPED);
while(curr != cnt && !ctx->isTestStopped())
{
if (curr > cnt)
{
ndbout_c("stopping: curr: %d cnt: %d", curr, cnt);
abort();
}
NdbSleep_MilliSleep(100);
curr= ctx->getProperty(NMR_SR_THREADS_STOPPED);
}
CHECK(ctx->isTestStopped() == false);
CHECK(startAll() == 0);
CHECK(waitClusterStarted(timeout) == 0);
cnt = ctx->getProperty(NMR_SR_VALIDATE_THREADS);
if (cnt)
{
ndbout << " -- Validating starts " << endl;
ctx->setProperty(NMR_SR_VALIDATE_THREADS_DONE, (Uint32)0);
ctx->setProperty(NMR_SR, NdbMixRestarter::SR_VALIDATING);
curr = ctx->getProperty(NMR_SR_VALIDATE_THREADS_DONE);
while (curr != cnt && !ctx->isTestStopped())
{
if (curr > cnt)
{
ndbout_c("validating: curr: %d cnt: %d", curr, cnt);
abort();
}
NdbSleep_MilliSleep(100);
curr = ctx->getProperty(NMR_SR_VALIDATE_THREADS_DONE);
}
ndbout << " -- Validating complete " << endl;
}
CHECK(ctx->isTestStopped() == false);
ctx->setProperty(NMR_SR, NdbMixRestarter::SR_RUNNING);
} while(0);
return result;
}
static
ndb_mgm_node_state*
select_node_to_stop(Vector<ndb_mgm_node_state>& nodes)
{
Uint32 i, j;
Vector<ndb_mgm_node_state*> alive_nodes;
for(i = 0; i<nodes.size(); i++)
{
ndb_mgm_node_state* node = &nodes[i];
if (node->node_status == NDB_MGM_NODE_STATUS_STARTED)
alive_nodes.push_back(node);
}
Vector<ndb_mgm_node_state*> victims;
// Remove those with one in node group
for(i = 0; i<alive_nodes.size(); i++)
{
int group = alive_nodes[i]->node_group;
for(j = 0; j<alive_nodes.size(); j++)
{
if (i != j && alive_nodes[j]->node_group == group)
{
victims.push_back(alive_nodes[i]);
break;
}
}
}
if (victims.size())
{
int victim = rand() % victims.size();
return victims[victim];
}
else
{
return 0;
}
}
static
ndb_mgm_node_state*
select_node_to_start(Vector<ndb_mgm_node_state>& nodes)
{
Uint32 i;
Vector<ndb_mgm_node_state*> victims;
for(i = 0; i<nodes.size(); i++)
{
ndb_mgm_node_state* node = &nodes[i];
if (node->node_status == NDB_MGM_NODE_STATUS_NOT_STARTED)
victims.push_back(node);
}
if (victims.size())
{
int victim = rand() % victims.size();
return victims[victim];
}
else
{
return 0;
}
}
void
NdbMixRestarter::setRestartTypeMask(Uint32 mask)
{
m_mask = mask;
}
int
NdbMixRestarter::runUntilStopped(NDBT_Context* ctx,
NDBT_Step* step,
Uint32 freq)
{
if (init(ctx, step))
return NDBT_FAILED;
while (!ctx->isTestStopped())
{
if (dostep(ctx, step))
return NDBT_FAILED;
NdbSleep_SecSleep(freq);
}
if (!finish(ctx, step))
return NDBT_FAILED;
return NDBT_OK;
}
int
NdbMixRestarter::runPeriod(NDBT_Context* ctx,
NDBT_Step* step,
Uint32 period, Uint32 freq)
{
if (init(ctx, step))
return NDBT_FAILED;
Uint32 stop = time(0) + period;
while (!ctx->isTestStopped() && (time(0) < stop))
{
if (dostep(ctx, step))
{
return NDBT_FAILED;
}
NdbSleep_SecSleep(freq);
}
if (finish(ctx, step))
{
return NDBT_FAILED;
}
ctx->stopTest();
return NDBT_OK;
}
int
NdbMixRestarter::init(NDBT_Context* ctx, NDBT_Step* step)
{
waitClusterStarted();
m_nodes = ndbNodes;
return 0;
}
int
NdbMixRestarter::dostep(NDBT_Context* ctx, NDBT_Step* step)
{
ndb_mgm_node_state* node = 0;
int action;
loop:
while(((action = (1 << (rand() % RTM_COUNT))) & m_mask) == 0);
switch(action){
case RTM_RestartCluster:
if (restart_cluster(ctx, step))
return NDBT_FAILED;
for (Uint32 i = 0; i<m_nodes.size(); i++)
m_nodes[i].node_status = NDB_MGM_NODE_STATUS_STARTED;
break;
case RTM_RestartNode:
case RTM_RestartNodeInitial:
case RTM_StopNode:
case RTM_StopNodeInitial:
{
if ((node = select_node_to_stop(m_nodes)) == 0)
goto loop;
if (action == RTM_RestartNode || action == RTM_RestartNodeInitial)
ndbout << "Restarting " << node->node_id << endl;
else
ndbout << "Stopping " << node->node_id << endl;
bool initial =
action == RTM_RestartNodeInitial || action == RTM_StopNodeInitial;
if (restartOneDbNode(node->node_id, initial, true, true))
return NDBT_FAILED;
if (waitNodesNoStart(&node->node_id, 1))
return NDBT_FAILED;
node->node_status = NDB_MGM_NODE_STATUS_NOT_STARTED;
if (action == RTM_StopNode || action == RTM_StopNodeInitial)
break;
else
goto start;
}
case RTM_StartNode:
if ((node = select_node_to_start(m_nodes)) == 0)
goto loop;
start:
ndbout << "Starting " << node->node_id << endl;
if (startNodes(&node->node_id, 1))
return NDBT_FAILED;
if (waitNodesStarted(&node->node_id, 1))
return NDBT_FAILED;
node->node_status = NDB_MGM_NODE_STATUS_STARTED;
break;
}
return NDBT_OK;
}
int
NdbMixRestarter::finish(NDBT_Context* ctx, NDBT_Step* step)
{
Vector<int> not_started;
{
ndb_mgm_node_state* node = 0;
while((node = select_node_to_start(m_nodes)))
{
not_started.push_back(node->node_id);
node->node_status = NDB_MGM_NODE_STATUS_STARTED;
}
}
if (not_started.size())
{
ndbout << "Starting stopped nodes " << endl;
if (startNodes(not_started.getBase(), not_started.size()))
return NDBT_FAILED;
if (waitClusterStarted())
return NDBT_FAILED;
}
return NDBT_OK;
}
template class Vector<ndb_mgm_node_state*>;
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment