BUG#3963

ndb/src/kernel/blocks/dbtc/Dbtc.hpp:
  let sendTCKEY_FAILCONF reset marker
ndb/src/kernel/blocks/dbtc/DbtcMain.cpp:
  1) let sendTCKEY_FAILCONF reset marker
  2) Restart marker scan on NF
  3) Don't send TCKEY_FAILCONF to 0
ndb/src/mgmapi/mgmapi.cpp:
  More info from mgmapi
ndb/test/run-test/daily-basic-tests.txt:
  drop tables to fix -n MassiveRollback
ndb/test/run-test/main.cpp:
  More error messages (and retries) on error

ndb/src/kernel/blocks/dbtc/Dbtc.hpp

@@ -1391,7 +1391,7 @@ private:
   void sendCompleteLqh(Signal* signal,
                        TcConnectRecord * const regTcPtr);
   void sendTCKEY_FAILREF(Signal* signal, const ApiConnectRecord *);
-  void sendTCKEY_FAILCONF(Signal* signal, const ApiConnectRecord *);
+  void sendTCKEY_FAILCONF(Signal* signal, ApiConnectRecord *);
   void checkStartTimeout(Signal* signal);
   void checkStartFragTimeout(Signal* signal);
   void timeOutFoundFragLab(Signal* signal, Uint32 TscanConPtr);

ndb/src/kernel/blocks/dbtc/DbtcMain.cpp

@@ -1048,13 +1048,12 @@ Dbtc::handleFailedApiNode(Signal* signal,
     apiConnectptr.i++;
     if (apiConnectptr.i > ((capiConnectFilesize / 3) - 1)) {
       jam();
-      capiConnectClosing[TapiFailedNode]--;
       /**
        * Finished with scanning connection record
        *
        * Now scan markers
        */
-      removeMarkerForFailedAPI(signal, TapiFailedNode, RNIL); // RNIL = first 
+      removeMarkerForFailedAPI(signal, TapiFailedNode, 0);
       return;
     }//if
   } while (TloopCount++ < 256);
@@ -1069,15 +1068,26 @@ Dbtc::removeMarkerForFailedAPI(Signal* signal,
                                Uint32 nodeId, 
                                Uint32 startBucket)
 {
-  CommitAckMarkerIterator iter;
-  if(startBucket == RNIL){
-    jam();
-    capiConnectClosing[nodeId]++;
-    m_commitAckMarkerHash.next(0, iter);
-  } else {
+  TcFailRecordPtr node_fail_ptr;
+  node_fail_ptr.i = 0;
+  ptrAss(node_fail_ptr, tcFailRecord);
+  if(node_fail_ptr.p->failStatus != FS_IDLE) {
     jam();
-    m_commitAckMarkerHash.next(startBucket, iter);
+    DEBUG("Restarting removeMarkerForFailedAPI");
+    /**
+     * TC take-over in progress
+     *   needs to restart as this
+     *   creates new markers
+     */
+    signal->theData[0] = TcContinueB::ZHANDLE_FAILED_API_NODE_REMOVE_MARKERS;
+    signal->theData[1] = nodeId;
+    signal->theData[2] = 0;
+    sendSignalWithDelay(cownref, GSN_CONTINUEB, signal, 500, 3);
+    return;
   }
+
+  CommitAckMarkerIterator iter;
+  m_commitAckMarkerHash.next(startBucket, iter);
   
   const Uint32 RT_BREAK = 256;
   for(Uint32 i = 0; i<RT_BREAK || iter.bucket == startBucket; i++){ 
@@ -1108,7 +1118,6 @@ Dbtc::removeMarkerForFailedAPI(Signal* signal,
       /**
        * Check so that the record is not still in use
        *
-       * (This can happen when NF and API Fail happens at the same time)
        */
       ApiConnectRecordPtr apiConnectPtr;
       apiConnectPtr.i = iter.curr.p->apiConnectPtr;
@@ -1120,9 +1129,8 @@ Dbtc::removeMarkerForFailedAPI(Signal* signal,
          *
          * Don't remove it, but continueb instead
          */
-        break;
+	break;
       }
-      
       sendRemoveMarkers(signal, iter.curr.p);
       m_commitAckMarkerHash.release(iter.curr);
       
@@ -7327,25 +7335,23 @@ Dbtc::sendTCKEY_FAILREF(Signal* signal, const ApiConnectRecord * regApiPtr){
 }
 
 void 
-Dbtc::sendTCKEY_FAILCONF(Signal* signal, const ApiConnectRecord * regApiPtr){
+Dbtc::sendTCKEY_FAILCONF(Signal* signal, ApiConnectRecord * regApiPtr){
   jam();
   TcKeyFailConf * const failConf = (TcKeyFailConf *)&signal->theData[0];
   
-  if(regApiPtr->commitAckMarker == RNIL){
-    jam();
-    failConf->apiConnectPtr = regApiPtr->ndbapiConnect;
-  } else {
-    jam();
-    failConf->apiConnectPtr = regApiPtr->ndbapiConnect | 1;
+  const Uint32 ref = regApiPtr->ndbapiBlockref;
+  const Uint32 marker = regApiPtr->commitAckMarker;
+  if(ref != 0){
+    failConf->apiConnectPtr = regApiPtr->ndbapiConnect | (marker != RNIL);
+    failConf->transId1 = regApiPtr->transid[0];
+    failConf->transId2 = regApiPtr->transid[1];
+    
+    sendSignal(regApiPtr->ndbapiBlockref,
+	       GSN_TCKEY_FAILCONF, signal, TcKeyFailConf::SignalLength, JBB);
   }
-  failConf->transId1 = regApiPtr->transid[0];
-  failConf->transId2 = regApiPtr->transid[1];
-  
-  sendSignal(regApiPtr->ndbapiBlockref,
-	     GSN_TCKEY_FAILCONF, signal, TcKeyFailConf::SignalLength, JBB);
+  regApiPtr->commitAckMarker = RNIL;
 }
 
-
 /*------------------------------------------------------------*/
 /*       THIS PART HANDLES THE ABORT PHASE IN THE CASE OF A   */
 /*       NODE FAILURE BEFORE THE COMMIT DECISION.             */

ndb/src/mgmapi/mgmapi.cpp

@@ -544,7 +544,7 @@ ndb_mgm_get_status(NdbMgmHandle handle)
     buf[strlen(buf)-1] = '\0';
 
   if(strcmp("node status", buf) != 0) {
-    SET_ERROR(handle, NDB_MGM_ILLEGAL_NODE_STATUS, "");
+    SET_ERROR(handle, NDB_MGM_ILLEGAL_NODE_STATUS, buf);
     return NULL;
   }
 

ndb/test/run-test/daily-basic-tests.txt

@@ -139,6 +139,10 @@ max-time: 500
 cmd: testBasicAsynch
 args: -n PkDeleteAsynch 
 
+max-time:
+cmd: ndb_drop_table
+args: T1 T2 T3 T4 T5 T6 T7 T8 T9 T10 T11 T12 T13 T14
+
 max-time: 500
 cmd: testBasic
 args: -n MassiveRollback T1 T6 T13 

ndb/test/run-test/main.cpp

@@ -614,11 +614,22 @@ wait_ndb(atrt_config& config, int goal){
     /**
      * 1) retreive current state
      */
-    state = ndb_mgm_get_status(handle);
-    if(state == 0){
-      g_logger.critical("Unable to poll db state");
-      return false;
-    }
+    state = 0;
+    do {
+      state = ndb_mgm_get_status(handle);
+      if(state == 0){
+	const int err = ndb_mgm_get_latest_error(handle);
+	g_logger.error("Unable to poll db state: %d %s %s",
+		       ndb_mgm_get_latest_error(handle),
+		       ndb_mgm_get_latest_error_msg(handle),
+		       ndb_mgm_get_latest_error_desc(handle));
+	if(err == NDB_MGM_SERVER_NOT_CONNECTED && connect_ndb_mgm(config)){
+	  g_logger.error("Reconnected...");
+	  continue;
+	}
+	return false;
+      }
+    } while(state == 0);
     NdbAutoPtr<void> tmp(state);
     
     min2 = goal;