ndb - bug#31701 Node failure with repl. wo/ load, can lead to endless out of order buckets

  Correct check for buffer/no buffer

storage/ndb/src/kernel/blocks/ERROR_codes.txt

@@ -11,7 +11,7 @@ Next CMVMI 9000
 Next BACKUP 10038
 Next DBUTIL 11002
 Next DBTUX 12008
-Next SUMA 13001
+Next SUMA 13034
 
 TESTING NODE FAILURE, ARBITRATION
 ---------------------------------

storage/ndb/src/kernel/blocks/suma/Suma.cpp

@@ -3650,6 +3650,8 @@ Suma::execSUB_GCP_COMPLETE_REP(Signal* signal)
   
   if(m_gcp_complete_rep_count && !c_subscriber_nodes.isclear())
   {
+    CRASH_INSERTION(13033);
+    
     NodeReceiverGroup rg(API_CLUSTERMGR, c_subscriber_nodes);
     sendSignal(rg, GSN_SUB_GCP_COMPLETE_REP, signal,
 	       SubGcpCompleteRep::SignalLength, JBB);
@@ -3669,8 +3671,8 @@ Suma::execSUB_GCP_COMPLETE_REP(Signal* signal)
   {
     if(m_active_buckets.get(i))
       continue;
-    
-    if(c_buckets[i].m_buffer_tail != RNIL)
+
+    if (!c_subscriber_nodes.isclear())
     {
       //Uint32* dst;
       get_buffer_ptr(signal, i, gci, 0);

storage/ndb/test/ndbapi/test_event.cpp

@@ -21,6 +21,7 @@
 #include <NdbAutoPtr.hpp>
 #include <NdbRestarter.hpp>
 #include <NdbRestarts.hpp>
+#include <signaldata/DumpStateOrd.hpp>
 
 #define GETNDB(ps) ((NDBT_NdbApiStep*)ps)->getNdb()
 
@@ -1758,6 +1759,85 @@ runInsertDeleteUntilStopped(NDBT_Context* ctx, NDBT_Step* step)
   return NDBT_OK;
 }
 
+int 
+runBug31701(NDBT_Context* ctx, NDBT_Step* step)
+{
+  int result = NDBT_OK;
+
+  NdbRestarter restarter;
+
+  if (restarter.getNumDbNodes() < 2){
+    ctx->stopTest();
+    return NDBT_OK;
+  }
+  // This should really wait for applier to start...10s is likely enough
+  NdbSleep_SecSleep(10);
+
+  int nodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
+
+  int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
+  if (restarter.dumpStateOneNode(nodeId, val2, 2))
+    return NDBT_FAILED;
+  
+  restarter.insertErrorInNode(nodeId, 13033);
+  if (restarter.waitNodesNoStart(&nodeId, 1))
+    return NDBT_FAILED;
+
+  if (restarter.startNodes(&nodeId, 1))
+    return NDBT_FAILED;
+
+  if (restarter.waitClusterStarted())
+    return NDBT_FAILED;
+
+  
+  int records = ctx->getNumRecords();
+  HugoTransactions hugoTrans(*ctx->getTab());
+  
+  if(ctx->getPropertyWait("LastGCI", ~(Uint32)0))
+  {
+    g_err << "FAIL " << __LINE__ << endl;
+    return NDBT_FAILED;
+  }
+
+  hugoTrans.clearTable(GETNDB(step), 0);
+  
+  if (hugoTrans.loadTable(GETNDB(step), 3*records, 1, true, 1) != 0){
+    g_err << "FAIL " << __LINE__ << endl;
+    return NDBT_FAILED;
+  }
+  
+  if (hugoTrans.pkDelRecords(GETNDB(step), 3*records, 1, true, 1) != 0){
+    g_err << "FAIL " << __LINE__ << endl;
+    return NDBT_FAILED;
+  }
+  if (hugoTrans.loadTable(GETNDB(step), records, 1, true, 1) != 0){
+    g_err << "FAIL " << __LINE__ << endl;
+    return NDBT_FAILED;
+  }
+  if (hugoTrans.pkUpdateRecords(GETNDB(step), records, 1, 1) != 0){
+    g_err << "FAIL " << __LINE__ << endl;
+    return NDBT_FAILED;
+  }
+  if (hugoTrans.pkUpdateRecords(GETNDB(step), records, 1, 1) != 0){
+    g_err << "FAIL " << __LINE__ << endl;
+    return NDBT_FAILED;
+  }
+  if (hugoTrans.pkUpdateRecords(GETNDB(step), records, 1, 1) != 0){
+    g_err << "FAIL " << __LINE__ << endl;
+    return NDBT_FAILED;
+  }
+  
+  ctx->setProperty("LastGCI", hugoTrans.m_latest_gci);
+  if(ctx->getPropertyWait("LastGCI", ~(Uint32)0))
+  {
+    g_err << "FAIL " << __LINE__ << endl;
+    return NDBT_FAILED;
+  }
+
+  ctx->stopTest();  
+  return NDBT_OK;
+}
+
 NDBT_TESTSUITE(test_event);
 TESTCASE("BasicEventOperation", 
 	 "Verify that we can listen to Events"
@@ -1887,6 +1967,14 @@ TESTCASE("Bug27169", ""){
   STEP(runRestarterLoop);
   FINALIZER(runDropEvent);
 }
+TESTCASE("Bug31701", ""){
+  INITIALIZER(runCreateEvent);
+  INITIALIZER(runCreateShadowTable);
+  STEP(runEventApplier);
+  STEP(runBug31701);
+  FINALIZER(runDropEvent);
+  FINALIZER(runDropShadowTable);
+}
 NDBT_TESTSUITE_END(test_event);
 
 int main(int argc, const char** argv){

storage/ndb/test/run-test/daily-basic-tests.txt

@@ -938,3 +938,7 @@ max-time: 600
 cmd: testNodeRestart
 args: -n Bug31525 T1
 
+max-time: 300
+cmd: test_event
+args: -n Bug31701 T1
+