slapos/test/collect: Extend tests to increase coverage

  On the tests for ConsumptionReportBase, it contains a port from a test
   on slapos.toolbox done by Alain Takoudjou.

slapos/collect/__init__.py

@@ -89,9 +89,9 @@ def do_collect(conf):
           user_dict[snapshot.username].append(snapshot)
     except (KeyboardInterrupt, SystemExit, NoSuchProcess):
       raise
-    
+    if conf.has_option("slapos", "collect_cache"):
       days_to_preserve = conf.get("slapos", "collect_cache")
-    if days_to_preserve is None:
+    else: 
       days_to_preserve = 15
     log_directory = "%s/var/data-log" % conf.get("slapos", "instance_root")
     mkdir_p(log_directory, 0o755)

slapos/collect/reporter.py

@@ -314,7 +314,7 @@ class PartitionReport(ConsumptionReportBase):
         }
         fdata.write(json.dumps(data_dict))
 
-  def buildJSONMonitorReport(self):
+  def buildJSONMonitorReport(self, date_scope=None, min_time=None, max_time=None):
 
     for user in self.user_list.values():
       location = os.path.join(user.path, ".slapgrid")
@@ -342,7 +342,8 @@ class PartitionReport(ConsumptionReportBase):
           ["date, io rw counter, io cycles counter, disk used"])
   
       process_result, memory_result, io_result = \
-              self.getPartitionConsumptionStatusList(user.name)
+              self.getPartitionConsumptionStatusList(
+                user.name, date_scope=date_scope, min_time=min_time, max_time=max_time)
   
       resource_status_dict = {}
       if process_result and process_result['total_process'] != 0.0:
@@ -366,7 +367,9 @@ class PartitionReport(ConsumptionReportBase):
       with open(status_file, 'w') as fp:
         fp.write(json.dumps(resource_status_dict))
   
-      resource_process_status_list = self.getPartitionProcessConsumptionList(user.name)
+      resource_process_status_list = self.getPartitionProcessConsumptionList(
+                user.name, date_scope=date_scope, min_time=min_time, max_time=max_time)
+
       if resource_process_status_list:
         with open(resource_file, 'w') as rf:
           rf.write(json.dumps(resource_process_status_list))
@@ -472,6 +475,8 @@ class ConsumptionReport(ConsumptionReportBase):
      return xml_report_path
 
   def _getAverageFromList(self, data_list):
+    if not data_list:
+      return 0
     return sum(data_list)/len(data_list)
 
   @withDB

slapos/tests/data/2019-10-07.xml

+<?xml version='1.0' encoding='utf-8'?><journal><transaction type="Sale Packing List"><title>Eco Information for COMP-192938 </title><start_date>2019-10-07 00:00:00</start_date><stop_date>2019-10-07 23:59:59</stop_date><reference>2019-10-07-global</reference><currency></currency><payment_mode></payment_mode><category></category><arrow type="Destination"/><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average</title><reference>COMP-192938</reference><quantity>76.4371428571</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Used Memory</title><reference>COMP-192938</reference><quantity>14211174517.0</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/zero_emission_ratio</resource><title>Zero Emission Ratio</title><reference>COMP-192938</reference><quantity>-1</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average for slapuser12</title><reference>slapuser12</reference><quantity>5.54571428571</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average for slapuser13</title><reference>slapuser13</reference><quantity>1.0</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average for slapuser10</title><reference>slapuser10</reference><quantity>18.1114285714</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average for slapuser9</title><reference>slapuser9</reference><quantity>0.822857142857</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average for slapuser2</title><reference>slapuser2</reference><quantity>5.79714285714</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average for slapuser3</title><reference>slapuser3</reference><quantity>0.971428571429</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/cpu_load_percent</resource><title>CPU Load Percent Average for slapuser6</title><reference>slapuser6</reference><quantity>1.39714285714</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Memory Used Average for slapuser12</title><reference>slapuser12</reference><quantity>1726.23069196</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Memory Used Average for slapuser13</title><reference>slapuser13</reference><quantity>793.577232143</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Memory Used Average for slapuser10</title><reference>slapuser10</reference><quantity>4311.40703125</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Memory Used Average for slapuser9</title><reference>slapuser9</reference><quantity>88.5420758929</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Memory Used Average for slapuser2</title><reference>slapuser2</reference><quantity>1784.5375</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Memory Used Average for slapuser3</title><reference>slapuser3</reference><quantity>370.342857143</quantity><price>0.0</price><VAT></VAT><category></category></movement><movement><resource>service_module/memory_used</resource><title>Memory Used Average for slapuser6</title><reference>slapuser6</reference><quantity>886.900558036</quantity><price>0.0</price><VAT></VAT><category></category></movement></transaction></journal>
\ No newline at end of file

slapos/tests/data/slapuser12/monitor_resource.status.json

+{"cpu_time": 59236.69, "cpu_percent": 3.2, "memory_rss": 1617.36, "memory_percent": 6.71, "io_rw_counter": 500668526592.0, "date": "2019-10-06 00:00:00", "total_process": 104, "disk_used": 0, "io_cycles_counter": 11844398924.0, "cpu_num_threads": 419.0}
\ No newline at end of file

slapos/tests/data/slapuser12/monitor_resource_io.data.json

+{"date": 1570495649.501284, "data": ["date, io rw counter, io cycles counter, disk used","2019-10-06 00:00:00, 5.00668526592e+11, 11844398924.0, 0"]}
\ No newline at end of file

slapos/tests/data/slapuser12/monitor_resource_memory.data.json

+{"date": 1570495649.501284, "data": ["date, memory used percent, memory used","2019-10-06 00:00:00, 6.71, 1617.36"]}
\ No newline at end of file

slapos/tests/data/slapuser12/monitor_resource_process.data.json

+{"date": 1570495649.501284, "data": ["date, total process, CPU percent, CPU time, CPU threads","2019-10-06 00:00:00, 104, 3.2, 59236.69, 419.0"]}
\ No newline at end of file

slapos/tests/data/slapuser19/monitor_resource.status.json

+{"cpu_time": 204815.91, "cpu_percent": 3.1, "memory_rss": 3642.55, "memory_percent": 15.1, "io_rw_counter": 3174929330176.0, "date": "2019-10-06 00:00:00", "total_process": 164, "disk_used": 0, "io_cycles_counter": 37490050852.0, "cpu_num_threads": 519.0}
\ No newline at end of file

slapos/tests/data/slapuser19/monitor_resource_io.data.json

+{"date": 1570495649.501284, "data": ["date, io rw counter, io cycles counter, disk used","2019-10-06 00:00:00, 3.17492933018e+12, 37490050852.0, 0"]}
\ No newline at end of file

slapos/tests/data/slapuser19/monitor_resource_memory.data.json

+{"date": 1570495649.501284, "data": ["date, memory used percent, memory used","2019-10-06 00:00:00, 15.1, 3642.55"]}
\ No newline at end of file

slapos/tests/data/slapuser19/monitor_resource_process.data.json

+{"date": 1570495649.501284, "data": ["date, total process, CPU percent, CPU time, CPU threads","2019-10-06 00:00:00, 164, 3.1, 204815.91, 519.0"]}
\ No newline at end of file

slapos/tests/test_slapproxy.py

@@ -361,6 +361,7 @@ class MasterMixin(BasicMixin, unittest.TestCase):
     Return response by server (a computer partition or an error)
     """
     rv = self._requestComputerPartition(*args, **kw)
+    time.sleep(0.01)
     requested_at = time.time()
     self.assertEqual(rv._status_code, 200)
     xml = rv.data