Provide more and often results

Instead of script being silent all the time provide results often, on
each try for runTestSuite.py.
Additionally print out full results of execution to the controlling
testnode, for better analysis.
in-vm-test script is adapted to follow such requirement.

software/cdn-test/in-vm-test

@@ -55,6 +55,15 @@ if [[ ! -s "$DEPLOYMENT_SCRIPT" ]] ; then
   echo "exit 1" > $DEPLOYMENT_SCRIPT
 fi
 
+function upload ()
+{
+  try=$1
+  LOG_FILE=$2
+  t=`date '+%Y%m%d%H%S'`
+  mv $LOG_FILE ${LOG_FILE}.$t
+  curl -q -X POST --data-urlencode "path=test-script-result/log-file.log.$t" --data-urlencode "content@${LOG_FILE}.$t" http://10.0.2.100/
+}
+
 try=1
 while true; do
   echo "$0: Try $try. Running '/bin/bash $DEPLOYMENT_SCRIPT'" >> $LOG_FILE 2>&1
@@ -62,20 +71,19 @@ while true; do
   result=$?
   if [ $result == 0 ] ; then
     echo "$0: Try $try. Script executed successfully." >> $LOG_FILE 2>&1
+    upload $try $LOG_FILE
     break
   fi
   if (( try > TRIES )) ; then
     echo "$0: Try $try. Amount of tries $TRIES exceeded, giving up." >> $LOG_FILE 2>&1
+    upload $try $LOG_FILE
     break
   fi
   # wait WAITTIME before checking the state
   echo "$0: Try $try. Sleeping $WAITTIME before retry." >> $LOG_FILE 2>&1
+  upload $try $LOG_FILE
   sleep $WAITTIME
   ((try++))
 done
 
-echo "$0: Try $try. Uploading log and exiting with $result." >> $LOG_FILE 2>&1
-t=`date '+%Y%m%d%H%S'`
-mv $LOG_FILE ${LOG_FILE}.$t
-curl -q -X POST --data-urlencode "path=test-script-result/log-file.log.$t" --data-urlencode "content@${LOG_FILE}.$t" http://10.0.2.100/
 exit $result

software/cdn-test/runTestSuite.py

@@ -81,9 +81,10 @@ def main():
     sleep_time = 10
     try_amount = (3600 * 10) / sleep_time
     try_num = 1
-    result_path = None
     start = time()
+    result_found = False
     while 1:
+      finished = False
       try_info = 'Try %s/%s.' % (try_num, try_amount)
       print(try_info + ' Waiting for data.')
       result_list = glob.glob(
@@ -98,39 +99,16 @@ def main():
       if len(result_list) > 0:
         print(try_info + ' Data found.')
         result_path = result_list[0]
-        break
-      if try_num > try_amount:
-        print(try_info + ' Time exceeded, data not found.')
-        break
-      try_num += 1
-      sleep(10)
-    if result_path is None:
-      status_dict['stdout'] = 'Test timed out and no result found.'
-      status_dict.update(
-          test_count=0,
-          skip_count=0,
-          failure_count=0,
-          error_count=0
-      )
-    else:
-      oldest_result = min(
-          (os.path.join(dirname, filename)
+        for result_file in (os.path.join(dirname, filename)
            for dirname, dirnames, filenames in os.walk(result_path)
            for filename in filenames
-           ),
-          key=lambda fn: os.stat(fn).st_mtime)
-      if oldest_result is None:
-        status_dict['stdout'] = 'Test finished but no result found.'
-        status_dict.update(
-            test_count=1,
-            skip_count=0,
-            failure_count=0,
-            error_count=1
-        )
-      else:
-        oldest_result = os.path.abspath(oldest_result)
-        status_dict['command'] = oldest_result
-        result = open(oldest_result).read()
+           ):
+          result_found = True
+          result_file = os.path.abspath(result_file)
+          status_dict['command'] = result_file
+          result = open(result_file).read()
+          print(try_info + 'Analysis of result %r:' % (result_file,))
+          print(try_info + result)
           if 'FATAL: all hosts have already failed -- aborting' in result:
             # failed
             status_dict.update(
@@ -139,6 +117,7 @@ def main():
                 failure_count=1,
                 error_count=0
             )
+            finished = False
           elif "\"msg\": \"[u'Build successful, connect to:', u'" in result:
             # success
             status_dict.update(
@@ -147,6 +126,8 @@ def main():
                 failure_count=0,
                 error_count=0
             )
+            finished = True
+            break
           else:
             # unknown
             status_dict.update(
@@ -157,9 +138,24 @@ def main():
             )
             status_dict['stdout'] = \
                 'Cannot find success or failure result in the output'
+            finished = False
           # send only part of the result
           status_dict['stderr'] = result[-8192:]
-
+      if finished:
+        break
+      if try_num > try_amount:
+        print(try_info + ' Time exceeded, data not found.')
+        break
+      try_num += 1
+      sleep(10)
+    if not result_found:
+      status_dict['stdout'] = 'Test timed out and no result found.'
+      status_dict.update(
+          test_count=0,
+          skip_count=0,
+          failure_count=0,
+          error_count=0
+      )
     end = time()
 
     test_result_line.stop(

software/cdn-test/software.cfg

@@ -42,7 +42,7 @@ recipe = slapos.recipe.download
 ignore-existing = true
 filename = in-vm-test
 url = ${:_profile_base_location_}/${:filename}
-md5sum = 13f10035a3008cffb55d23a7dd069861
+md5sum = e96e1338afa99381cef55093e1f5684d
 mode = 0644
 download-only = true
 on-update = true