monitor: Cleanup stale history.json files

Additionally improve python3 support as it is required for proper
cleanup procedure.

/reviewed-on nexedi/slapos.toolbox!74

slapos/monitor/globalstate.py

@@ -100,10 +100,10 @@ def generateMonitoringData(config, public_folder, private_folder, public_url,
     private_url, feed_url):
   feed_output = os.path.join(public_folder, 'feed')
   # search for all status files
-  file_list = filter(
+  file_list = list(filter(
     os.path.isfile,
     glob.glob("%s/promise/*.status.json" % public_folder)
-  )
+  ))
 
   promises_status_file = os.path.join(private_folder, '_promise_status')
   previous_state_dict = {}
@@ -123,6 +123,21 @@ def generateMonitoringData(config, public_folder, private_folder, public_url,
       except ValueError:
         pass
 
+  # clean up stale history files
+  expected_history_json_name_list = [
+    os.path.basename(q).replace('status.json', 'history.json') for q in file_list]
+  cleanup_history_json_path_list = []
+  for history_json_name in [q for q in os.listdir(public_folder) if q.endswith('history.json')]:
+    if history_json_name not in expected_history_json_name_list:
+      cleanup_history_json_path_list.append(os.path.join(public_folder, history_json_name))
+  for cleanup_path in cleanup_history_json_path_list:
+    try:
+      os.unlink(cleanup_path)
+    except Exception:
+      print('ERROR: Failed to remove stale %s' % (cleanup_path,))
+    else:
+      print('OK: Removed stale %s' % (cleanup_path,))
+
   for file in file_list:
     try:
       with open(file, 'r') as temp_file:

slapos/monitor/monitor.py

@@ -4,6 +4,10 @@
 from __future__ import print_function
 
 import sys
+try:
+  import errno
+except ImportError:
+  from os import errno
 import os
 import stat
 import json
@@ -53,7 +57,7 @@ def mkdirAll(path):
   try:
     os.makedirs(path)
   except OSError as e:
-    if e.errno == os.errno.EEXIST and os.path.isdir(path):
+    if e.errno == errno.EEXIST and os.path.isdir(path):
       pass
     else: raise
 
@@ -67,7 +71,7 @@ def createSymlink(source, destination):
   try:
     os.symlink(source, destination)
   except OSError as e:
-    if e.errno != os.errno.EEXIST:
+    if e.errno != errno.EEXIST:
       raise
 
 class Monitoring(object):
@@ -201,7 +205,7 @@ class Monitoring(object):
             mkdirAll(dirname)  # could also raise OSError
             os.symlink(path, os.path.join(dirname, os.path.basename(path)))
           except OSError as e:
-            if e.errno != os.errno.EEXIST:
+            if e.errno != errno.EEXIST:
               raise
 
   def getMonitorTitleFromUrl(self, monitor_url):

slapos/test/monitor/testglobalstate.py

@@ -166,8 +166,22 @@ exit %(code)s
     self.assertTrue(os.path.exists(os.path.join(self.output_dir, 'promise_4.status.json')))
 
     os.symlink(self.output_dir, '%s/public/promise' % self.base_dir)
+    # create files for cleanup
+    must_stay_public = os.path.join(self.public_dir, 'must_stay')
+    must_unlink_public = os.path.join(self.public_dir, 'must_unlink.history.json')
+    for f in [must_stay_public, must_unlink_public]:
+      with open(f, 'w') as fh:
+        fh.write('data')
     # generate instance state files
+    document_list_file = os.path.join(self.public_dir, '_document_list')
+    with open(document_list_file, 'a+') as fh:
+      fh.write('must_unlink.history')
+
     globalstate.run(self.monitor_config_file)
+    with open(document_list_file) as fh:
+      self.assertNotIn('must_unlink.history', fh.read())
+    self.assertTrue(os.path.exists(must_stay_public))
+    self.assertFalse(os.path.exists(must_unlink_public))
     self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'feed')))
     self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'monitor.global.json')))
     self.assertTrue(os.path.exists(os.path.join(self.private_dir, 'monitor.global.json')))