bug#11942

ndb/include/kernel/signaldata/AlterTable.hpp:
  Add error code for backup in progress
ndb/include/kernel/signaldata/DictTabInfo.hpp:
  Add backup state
ndb/include/kernel/signaldata/DropTable.hpp:
  Add error code for backup in progress
ndb/include/ndbapi/NdbDictionary.hpp:
  Add backup state
ndb/src/kernel/blocks/backup/Backup.cpp:
  1) remove invalid require (util_sequence_ref) crash
  2) Don't backup objects dropping/creating
  3) set correct error code on backup fragment ref (crash)
  4) save TrigAttrInfo header when getting log full (crash)
  5) lock/unlock tables during backup
ndb/src/kernel/blocks/dbdict/Dbdict.cpp:
  1) add mutex lock/unlock as part of drop/alter table
  2) add lock/unlock for backup
  3) remove TC from backup trigger loop
ndb/src/kernel/blocks/dbdict/Dbdict.hpp:
  Add BACKUP_ONGOING state
ndb/src/ndbapi/NdbDictionaryImpl.cpp:
  Add backup state
ndb/src/ndbapi/ndberror.c:
  Add error code for backup in progress
ndb/test/ndbapi/testBackup.cpp:
  Add testcase for testBackup -n BackupDDL
ndb/tools/drop_index.cpp:
  Fix ndb_drop_index
ndb/tools/listTables.cpp:
  Print of backup state

ndb/include/kernel/signaldata/AlterTable.hpp

@@ -128,7 +128,8 @@ public:
     RecordTooBig = 738,
     InvalidPrimaryKeySize  = 739,
     NullablePrimaryKey = 740,
-    UnsupportedChange = 741
+    UnsupportedChange = 741,
+    BackupInProgress = 746
   };
 
 private:

ndb/include/kernel/signaldata/DictTabInfo.hpp

@@ -209,6 +209,7 @@ public:
     StateBuilding = 2,
     StateDropping = 3,
     StateOnline = 4,
+    StateBackup = 5,
     StateBroken = 9
   };
 

ndb/include/kernel/signaldata/DropTable.hpp

@@ -57,7 +57,8 @@ public:
     NoSuchTable         = 709,
     InvalidTableVersion = 241,
     DropInProgress      = 283,
-    NoDropTableRecordAvailable = 1229
+    NoDropTableRecordAvailable = 1229,
+    BackupInProgress = 745
   };
 };
 

ndb/include/ndbapi/NdbDictionary.hpp

@@ -118,6 +118,7 @@ public:
       StateBuilding = 2,      ///< Building, not yet usable
       StateDropping = 3,      ///< Offlining or dropping, not usable
       StateOnline = 4,        ///< Online, usable
+      StateBackup = 5,        ///< Online, being backuped, usable
       StateBroken = 9         ///< Broken, should be dropped and re-created
     };
 

ndb/src/kernel/blocks/backup/Backup.cpp

@@ -921,7 +921,6 @@ Backup::execUTIL_SEQUENCE_REF(Signal* signal)
   jamEntry();
   UtilSequenceRef * utilRef = (UtilSequenceRef*)signal->getDataPtr();
   ptr.i = utilRef->senderData;
-  ndbrequire(ptr.i == RNIL);
   c_backupPool.getPtr(ptr);
   ndbrequire(ptr.p->masterData.gsn == GSN_UTIL_SEQUENCE_REQ);
   sendBackupRef(signal, ptr, BackupRef::SequenceFailure);
@@ -2418,10 +2417,16 @@ Backup::execLIST_TABLES_CONF(Signal* signal)
     jam();
     Uint32 tableId = ListTablesConf::getTableId(conf->tableData[i]);
     Uint32 tableType = ListTablesConf::getTableType(conf->tableData[i]);
+    Uint32 state= ListTablesConf::getTableState(conf->tableData[i]);
     if (!DictTabInfo::isTable(tableType) && !DictTabInfo::isIndex(tableType)){
       jam();
       continue;
     }//if
+    if (state != DictTabInfo::StateOnline)
+    {
+      jam();
+      continue;
+    }//if
     TablePtr tabPtr;
     ptr.p->tables.seize(tabPtr);
     if(tabPtr.i == RNIL) {
@@ -2791,10 +2796,19 @@ Backup::execGET_TABINFO_CONF(Signal* signal)
 
   TablePtr tmp = tabPtr;
   ptr.p->tables.next(tabPtr);
-  if(DictTabInfo::isIndex(tmp.p->tableType)){
+  if(DictTabInfo::isIndex(tmp.p->tableType))
+  {
+    jam();
     ptr.p->tables.release(tmp);
   }
-  
+  else
+  {
+    jam();
+    signal->theData[0] = tmp.p->tableId;
+    signal->theData[1] = 1; // lock
+    EXECUTE_DIRECT(DBDICT, GSN_BACKUP_FRAGMENT_REQ, signal, 2);
+  }
+
   if(tabPtr.i == RNIL) {
     jam();
     
@@ -3575,7 +3589,7 @@ Backup::backupFragmentRef(Signal * signal, BackupFilePtr filePtr)
   ref->backupId = ptr.p->backupId;
   ref->backupPtr = ptr.i;
   ref->nodeId = getOwnNodeId();
-  ref->errorCode = ptr.p->errorCode;
+  ref->errorCode = filePtr.p->errorCode;
   sendSignal(ptr.p->masterRef, GSN_BACKUP_FRAGMENT_REF, signal,
 	     BackupFragmentRef::SignalLength, JBB);
 }
@@ -3836,6 +3850,8 @@ Backup::execTRIG_ATTRINFO(Signal* signal) {
        !buf.getWritePtr(&dst, trigPtr.p->maxRecordSize)) 
     {
       jam();
+      Uint32 save[TrigAttrInfo::StaticLength];
+      memcpy(save, signal->getDataPtr(), 4*TrigAttrInfo::StaticLength);
       BackupRecordPtr ptr;
       c_backupPool.getPtr(ptr, trigPtr.p->backupPtr);
       trigPtr.p->errorCode = AbortBackupOrd::LogBufferFull;
@@ -3846,6 +3862,8 @@ Backup::execTRIG_ATTRINFO(Signal* signal) {
       ord->senderData= ptr.i;
       sendSignal(ptr.p->masterRef, GSN_ABORT_BACKUP_ORD, signal, 
 		 AbortBackupOrd::SignalLength, JBB);
+
+      memcpy(signal->getDataPtrSend(), save, 4*TrigAttrInfo::StaticLength);
       return;
     }//if
         
@@ -3995,6 +4013,17 @@ Backup::execSTOP_BACKUP_REQ(Signal* signal)
     gcp->StopGCP       = htonl(stopGCP - 1);
     filePtr.p->operation.dataBuffer.updateWritePtr(gcpSz);
   }
+
+  {
+    TablePtr tabPtr;
+    for(ptr.p->tables.first(tabPtr); tabPtr.i != RNIL;
+	ptr.p->tables.next(tabPtr))
+    {
+      signal->theData[0] = tabPtr.p->tableId;
+      signal->theData[1] = 0; // unlock
+      EXECUTE_DIRECT(DBDICT, GSN_BACKUP_FRAGMENT_REQ, signal, 2);
+    }
+  }
   
   closeFiles(signal, ptr);
 }
@@ -4338,6 +4367,11 @@ Backup::cleanup(Signal* signal, BackupRecordPtr ptr)
       }//if
       tabPtr.p->triggerIds[j] = ILLEGAL_TRIGGER_ID;
     }//for
+    {
+      signal->theData[0] = tabPtr.p->tableId;
+      signal->theData[1] = 0; // unlock
+      EXECUTE_DIRECT(DBDICT, GSN_BACKUP_FRAGMENT_REQ, signal, 2);
+    }
   }//for
 
   BackupFilePtr filePtr;
@@ -4352,9 +4386,6 @@ Backup::cleanup(Signal* signal, BackupRecordPtr ptr)
   }//for
 
   ptr.p->files.release();
-  ptr.p->tables.release();
-  ptr.p->triggers.release();
-
   ptr.p->tables.release();
   ptr.p->triggers.release();
   ptr.p->pages.release();

ndb/src/kernel/blocks/dbdict/Dbdict.hpp

@@ -149,8 +149,6 @@ public:
     /**    Pointer to last attribute in table */
     Uint32 lastAttribute;
 
-    /*    Temporary record used during add/drop table */
-    Uint32 myConnect;
 #ifdef HAVE_TABLE_REORG    
     /*    Second table used by this table (for table reorg) */
     Uint32 secondTable;
@@ -174,7 +172,8 @@ public:
       CHECKED = 3,
       DEFINED = 4,
       PREPARE_DROPPING = 5,
-      DROPPING = 6
+      DROPPING = 6,
+      BACKUP_ONGOING = 7
     };
     TabState tabState;
 
@@ -502,6 +501,8 @@ private:
   void execBUILDINDXCONF(Signal* signal);
   void execBUILDINDXREF(Signal* signal);
 
+  void execBACKUP_FRAGMENT_REQ(Signal*);
+
   // Util signals used by Event code
   void execUTIL_PREPARE_CONF(Signal* signal);
   void execUTIL_PREPARE_REF (Signal* signal);
@@ -894,6 +895,8 @@ private:
     
     Uint32 m_errorCode;
     void setErrorCode(Uint32 c){ if(m_errorCode == 0) m_errorCode = c;}
+
+    MutexHandle2<BACKUP_DEFINE_MUTEX> m_define_backup_mutex;
     
     /**
      * When sending stuff around
@@ -1908,6 +1911,7 @@ private:
 
   bool getIsFailed(Uint32 nodeId) const;
 
+  void dropTable_backup_mutex_locked(Signal* signal, Uint32, Uint32);
   void dropTableRef(Signal * signal, DropTableReq *, DropTableRef::ErrorCode);
   void printTables(); // For debugging only
   int handleAlterTab(AlterTabReq * req,
@@ -1918,6 +1922,7 @@ private:
 			Uint32 changeMask, 
 			Uint32 tableId,
 			CreateTableRecord * regAlterTabPtr);
+  void alterTable_backup_mutex_locked(Signal* signal, Uint32, Uint32);
   void alterTableRef(Signal * signal, 
 		     AlterTableReq *, AlterTableRef::ErrorCode, 
 		     ParseDictTabInfoRecord* parseRecord = NULL);

ndb/src/ndbapi/NdbDictionaryImpl.cpp

@@ -1150,6 +1150,7 @@ objectStateMapping[] = {
   { DictTabInfo::StateBuilding,      NdbDictionary::Object::StateBuilding },
   { DictTabInfo::StateDropping,      NdbDictionary::Object::StateDropping },
   { DictTabInfo::StateOnline,        NdbDictionary::Object::StateOnline },
+  { DictTabInfo::StateBackup,        NdbDictionary::Object::StateBackup },
   { DictTabInfo::StateBroken,        NdbDictionary::Object::StateBroken }, 
   { -1, -1 }
 };

ndb/src/ndbapi/ndberror.c

@@ -312,6 +312,8 @@ ErrorBundle ErrorCodes[] = {
   { 742,  SE, "Unsupported attribute type in index" },
   { 743,  SE, "Unsupported character set in table or index" },
   { 744,  SE, "Character string is invalid for given character set" },
+  { 745,  SE, "Unable to drop table as backup is in progress" },
+  { 746,  SE, "Unable to alter table as backup is in progress" },
   { 241,  SE, "Invalid schema object version" },
   { 283,  SE, "Table is being dropped" },
   { 284,  SE, "Table not defined in transaction coordinator" },

ndb/test/ndbapi/testBackup.cpp

@@ -138,6 +138,61 @@ int runBackupOne(NDBT_Context* ctx, NDBT_Step* step){
   return NDBT_OK;
 }
 
+int
+runBackupLoop(NDBT_Context* ctx, NDBT_Step* step){
+  NdbBackup backup(GETNDB(step)->getNodeId()+1);
+  unsigned backupId = 0;
+  
+  int loops = ctx->getNumLoops();
+  while(!ctx->isTestStopped() && loops--)
+  {
+    if (backup.start(backupId) == -1)
+    {
+      sleep(1);
+      loops++;
+    }
+    else
+    {
+      sleep(3);
+    }
+  }
+
+  ctx->stopTest();
+  return NDBT_OK;
+}
+
+int
+runDDL(NDBT_Context* ctx, NDBT_Step* step){
+  Ndb* pNdb= GETNDB(step);
+  NdbDictionary::Dictionary* pDict = pNdb->getDictionary();
+  
+  const int tables = NDBT_Tables::getNumTables();
+  while(!ctx->isTestStopped())
+  {
+    const int tab_no = rand() % (tables);
+    NdbDictionary::Table tab = *NDBT_Tables::getTable(tab_no);
+    BaseString name= tab.getName();
+    name.appfmt("-%d", step->getStepNo());
+    tab.setName(name.c_str());
+    if(pDict->createTable(tab) == 0)
+    {
+      HugoTransactions hugoTrans(* pDict->getTable(name.c_str()));
+      if (hugoTrans.loadTable(pNdb, 10000) != 0){
+	return NDBT_FAILED;
+      }
+      
+      while(pDict->dropTable(tab.getName()) != 0 &&
+	    pDict->getNdbError().code != 4009)
+	g_err << pDict->getNdbError() << endl;
+      
+      sleep(1);
+
+    }
+  }
+  return NDBT_OK;
+}
+
+
 int runRestartInitial(NDBT_Context* ctx, NDBT_Step* step){
   NdbRestarter restarter;
 
@@ -417,6 +472,15 @@ TESTCASE("BackupOne",
   VERIFIER(runVerifyOne);
   FINALIZER(runClearTable);
 }
+TESTCASE("BackupDDL", 
+	 "Test that backup and restore works on with DDL ongoing\n"
+	 "1. Backups and DDL (create,drop,table.index)"){
+  INITIALIZER(runLoadTable);
+  STEP(runBackupLoop);
+  STEP(runDDL);
+  STEP(runDDL);
+  FINALIZER(runClearTable);
+}
 TESTCASE("BackupBank", 
 	 "Test that backup and restore works during transaction load\n"
 	 " by backing up the bank"

ndb/tools/drop_index.cpp

@@ -35,7 +35,7 @@ static struct my_option my_long_options[] =
 static void usage()
 {
   char desc[] = 
-    "<indexname>+\n"\
+    "[<table> <index>]+\n"\
     "This program will drop index(es) in Ndb\n";
   ndb_std_print_version();
   my_print_help(my_long_options);
@@ -73,10 +73,10 @@ int main(int argc, char** argv){
     ndbout << "Waiting for ndb to become ready..." << endl;
   
   int res = 0;
-  for(int i = 0; i<argc; i++){
-    ndbout << "Dropping index " <<  argv[i] << "...";
+  for(int i = 0; i+1<argc; i += 2){
+    ndbout << "Dropping index " << argv[i] << "/" << argv[i+1] << "...";
     int tmp;
-    if((tmp = MyNdb.getDictionary()->dropIndex(argv[i], 0)) != 0){
+    if((tmp = MyNdb.getDictionary()->dropIndex(argv[i+1], argv[i])) != 0){
       ndbout << endl << MyNdb.getDictionary()->getNdbError() << endl;
       res = tmp;
     } else {

ndb/tools/listTables.cpp

@@ -131,6 +131,9 @@ list(const char * tabname,
         case NdbDictionary::Object::StateOnline:
             strcpy(state, "Online");
             break;
+        case NdbDictionary::Object::StateBackup:
+	    strcpy(state, "Backup");
+            break;
         case NdbDictionary::Object::StateBroken:
             strcpy(state, "Broken");
             break;