Commit dda88cd5 authored by Justin's avatar Justin Committed by Xavier Thompson

promise/plugin: add check_ram_usage promise

Co-authored-by: Xavier Thompson's avatarXavier Thompson <xavier.thompson@nexedi.com>
parent 00ffdc8e
import json
import os
import psutil
import time
from psutil._common import bytes2human
from .util import JSONPromise
from zope.interface import implementer
from slapos.grid.promise import interface
@implementer(interface.IPromise)
class RunPromise(JSONPromise):
def __init__(self, config):
super(RunPromise, self).__init__(config)
self.setPeriodicity(minute=2)
self.last_avg_ram_file = self.getConfig(
'last-avg-ram-file', 'last_avg')
def sense(self):
promise_success = True
# Get reference values
min_threshold_ram = float(self.getConfig('min-threshold-ram', 500e6)) # ≈500 MB
min_avg_ram = float(self.getConfig('min-avg-ram', 1e9)) # ≈1 GB
avg_ram_period_sec = int(self.getConfig('avg-ram-period-sec', 0))
if avg_ram_period_sec:
avg_ram_period = avg_ram_period_sec
else:
avg_ram_period = 60 * int(self.getConfig('avg-ram-period', 5))
# Get current RAM usage
ram_data = psutil.virtual_memory()
# Check with min threshold and log error if below it
if ram_data.available <= min_threshold_ram:
self.logger.error("RAM usage reached critical threshold: %7s "\
" (threshold is %7s)" % (bytes2human(ram_data.available), bytes2human(min_threshold_ram)))
promise_success = False
# Log RAM usage
data = json.dumps({'available_ram': ram_data.available})
self.json_logger.info("RAM data", extra={'data': data})
# Get last timestamp (i.e. last modification) of log file
try:
t = os.path.getmtime(self.last_avg_ram_file)
except OSError:
t = 0
# Get last available RAM from log file since avg_ram_period
if (time.time() - t) > avg_ram_period:
open(self.last_avg_ram_file, 'w').close()
temp_list = self.getJsonLogDataInterval(avg_ram_period)
if temp_list:
avg_ram = sum(map(lambda x: x['available_ram'], temp_list)) / len(temp_list)
if avg_ram < min_avg_ram:
self.logger.error("Average RAM usage over the last %s seconds "\
"reached threshold: %7s (threshold is %7s)"
% (avg_ram_period, bytes2human(avg_ram), bytes2human(min_avg_ram)))
promise_success = False
else:
self.logger.error("Couldn't read available RAM from log")
promise_success = False
if promise_success:
self.logger.info("RAM usage OK")
def test(self):
"""
Called after sense() if the instance is still converging.
Returns success or failure based on sense results.
In this case, fail if the previous sensor result is negative.
"""
return self._test(result_count=1, failure_amount=1)
def anomaly(self):
"""
Called after sense() if the instance has finished converging.
Returns success or failure based on sense results.
Failure signals the instance has diverged.
In this case, fail if two out of the last three results are negative.
"""
return self._anomaly(result_count=3, failure_amount=2)
# -*- coding: utf-8 -*-
##############################################################################
# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
##############################################################################
import mock
import os
import time
from collections import namedtuple
from slapos.grid.promise import PromiseError
from slapos.promise.plugin.check_ram_usage import RunPromise
from . import TestPromisePluginMixin
class TestCheckRamUsage(TestPromisePluginMixin):
promise_name = "monitor-ram-usage.py"
def setUp(self):
super(TestCheckRamUsage, self).setUp()
def writePromise(self, **kw):
super(TestCheckRamUsage, self).writePromise(self.promise_name,
"from %s import %s\nextra_config_dict = %r\n"
% (RunPromise.__module__, RunPromise.__name__, kw))
def runPromise(self, summary, failed=False):
self.configureLauncher(enable_anomaly=True, force=True)
with mock.patch('psutil.virtual_memory', return_value=summary):
if failed:
self.assertRaises(PromiseError, self.launcher.run)
else:
self.launcher.run()
result = self.getPromiseResult(self.promise_name)['result']
self.assertEqual(result['failed'], failed)
return result['message']
def test_ram_ok(self):
message = "RAM usage OK"
ram_data = namedtuple('ram_data', ['available'])
available_ram = {'available':1e9}
self.writePromise(**{
'last-avg-ram-file':'last_avg_ram_file',
'min-threshold-ram': 500e6, # 500MB
'min-avg-ram': 100,
})
self.assertEqual(message, self.runPromise(ram_data(**available_ram)))
def test_ram_below_threshold_nok(self):
message = "RAM usage reached critical threshold: 190.7M (threshold is 476.8M)"
ram_data = namedtuple('ram_data', ['available'])
available_ram = {'available': 200e6}
self.writePromise(**{
'last-avg-ram-file':'last_avg_ram_file',
'min-threshold-ram': 500e6, # ≈500MB
'min-avg-ram': 100,
})
self.assertEqual(message, self.runPromise(ram_data(**available_ram)))
def test_ram_below_average_nok(self):
message = "Average RAM usage over the last 1 seconds reached threshold: 100.5B (threshold is 200.0B)"
ram_data = namedtuple('ram_data', ['available'])
available_ram = {'available': 200}
self.writePromise(**{
'last-avg-ram-file':'last_avg_ram_file',
'min-threshold-ram': 0,
'min-avg-ram': 200,
'avg-ram-period-sec': 1,
})
m = self.runPromise(ram_data(**{'available': 1}))
time.sleep(1)
self.assertEqual(message, self.runPromise(ram_data(**available_ram), failed=True))
if __name__ == '__main__':
unittest.main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment