agent: Be more tolerant for eventual network problems

  Retry if git command fail (to fetch for example) and also retry if slapos report a connection error

slapos/agent/agent.py

@@ -133,7 +133,19 @@ def getAndUpdateFullRevisionList(node_test_suite, working_directory,
        branch=branch, log=logger.info, process_manager=process_manager,
        working_directory=working_directory,
        url=vcs_repository["url"])
-    updater.checkout()
+
+    retry = 10
+    while retry:
+      try:
+        updater.checkout()
+      except SubprocessError:
+        # Wait a bit and try again
+        time.sleep(30)
+        retry -= 1
+        continue
+      finally:
+        break 
+
     revision = "-".join(updater.getRevision())
     full_revision_list.append('%s=%s' % (repository_id, revision))
   node_test_suite['revision'] = ','.join(full_revision_list)

slapos/agent/tester.py

@@ -46,6 +46,8 @@ def retryOnNetworkFailure(func):
         return func(*args, **kwargs)
       except SAFE_RPC_EXCEPTION_LIST, e:
         print 'Network failure: %s , %s' % (sys.exc_info(), e)
+      except slapos.slap.slap.ConnectionError, e:
+        print 'Network failure: %s , %s' % (sys.exc_info(), e)
 
       print 'Retry method %s in %i seconds' % (func, retry_time)
       time.sleep(retry_time)