Commit 4c3b7a44 authored by Joanne Hugé's avatar Joanne Hugé Committed by lu.xu

software: add promise testing SR

parent b38e89d2
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# But avoid directories, they are not portable.
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template]
filename = instance.cfg
md5sum = 23acb8fc6465bf0ccdfdd366c48a2688
[template-default]
_update_hash_filename_ = instance-default.jinja2.cfg
md5sum = 97c4a1d7b1e72728ad66f3e710180b27
[cpu-promise]
_update_hash_filename_ = promise/check_cpu_temperature.py
md5sum = e8fc789eb7e7e3b054e921b0292c806f
{
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"type": "object",
"properties": {
"max_temp": {
"default": "80",
"title": "Max Temp",
"description": "Max Temp",
"type": "string"
}
}
}
[buildout]
parts =
directory
cpu-promise
publish-connection-information
extends = {{ monitor_template }}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = {{ slap_connection['computer-id'] }}
partition = {{ slap_connection['partition-id'] }}
url = {{ slap_connection['server-url'] }}
key = {{ slap_connection['key-file'] }}
cert = {{ slap_connection['cert-file'] }}
[directory]
recipe = slapos.cookbook:mkdirectory
software = {{ buildout_directory }}
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
etc = ${:home}/etc
bin = ${:home}/bin
run = ${:var}/run
script = ${:etc}/run
service = ${:etc}/service
promise = ${:etc}/promise
log = ${:var}/log
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
<= monitor-publish
[cpu-promise]
<= monitor-promise-base
promise = check_cpu_temperature
name = ${:_buildout_section_name_}.py
config-testing = {{ slapparameter_dict.get("testing", False) }}
config-max-spot-temp = 90
config-max-avg-temp = 80
config-avg-temp-duration = 15
config-last-avg-computation-file = ${directory:var}/promise_cpu_last_avg_computation_file
[buildout]
parts =
switch-softwaretype
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
output = $${buildout:directory}/$${:filename}
extra-context =
context =
import json_module json
key eggs_directory buildout:eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
raw buildout_directory ${buildout:directory}
section slap_connection slap-connection
key slapparameter_dict slap-configuration:configuration
$${:extra-context}
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
default = dynamic-template-default:output
RootSoftwareInstance = $${:default}
[dynamic-template-default]
< = jinja2-template-base
url = ${template-default:target}
filename = instance-default.cfg
extensions = jinja2.ext.do
extra-context =
raw monitor_template ${monitor2-template:output}
raw cpu_promise ${cpu-promise:target}
import errno
import json
import logging
import os
import psutil
import socket
import time
from dateutil import parser
from zope.interface import implementer
from slapos.grid.promise import interface
from slapos.grid.promise.generic import GenericPromise
# Get all data in the last "interval" seconds from JSON log
def get_data_interval(log, interval):
log_number = 0
latest_timestamp = 0
data_list = []
while True:
try:
f = open("{}.{}".format(log, log_number) if log_number else log, "rb")
except OSError:
return data_list
try:
f.seek(0, os.SEEK_END)
while True:
try:
while f.seek(-2, os.SEEK_CUR) and f.read(1) != b'\n':
pass
except OSError:
break
pos = f.tell()
l = json.loads(f.readline().decode().replace("'", '"'))
timestamp = parser.parse(l['time'])
data_list.append(l['data'])
if not latest_timestamp:
latest_timestamp = timestamp
if (latest_timestamp - timestamp).total_seconds() > interval:
return data_list
f.seek(pos, os.SEEK_SET)
finally:
f.close()
log_number += 1
@implementer(interface.IPromise)
class RunPromise(GenericPromise):
def __init__(self, config):
self.__name = config.get('name', None)
self.__log_folder = config.get('log-folder', None)
super(RunPromise, self).__init__(config)
self.setPeriodicity(minute=2)
self.last_avg_computation_file = self.getConfig(
'last-avg-computation-file', '')
self.__title = os.path.splitext(self.__name)[0]
self.__log_file = os.path.join(self.__log_folder, '%s.json.log' % self.__title)
self.json_logger = logging.getLogger('json_logger')
self.json_logger.setLevel(logging.INFO)
handler = logging.FileHandler(self.__log_file)
formatter = logging.Formatter('{"time": "%(asctime)s", "log_level": '\
'"%(levelname)s", "message": "%(message)s", "data": %(data)s}')
handler.setFormatter(formatter)
self.json_logger.addHandler(handler)
def sense(self):
promise_success = True
max_spot_temp = float(self.getConfig('max-spot-temp', 90))
max_avg_temp = float(self.getConfig('max-avg-temp', 80))
avg_temp_duration = 60 * int(self.getConfig('avg-temp-duration', 5))
testing = self.getConfig('testing') == "True"
# Get current temperature
# TODO: use mock
if testing:
from random import randint
cpu_temp = randint(40, 75)
else:
data = psutil.sensors_temperatures()
cpu_temp = data['coretemp'][0][1]
if cpu_temp > max_spot_temp:
self.logger.error("Temperature reached critical threshold: %s degrees "\
"celsius (threshold is %s degrees celsius)" % (cpu_temp, max_spot_temp))
promise_success = False
# Log temperature
data = json.dumps({'cpu_temperature': cpu_temp})
self.json_logger.info("Temperature data", extra={'data': data})
# Computer average temperature
avg_computation_period = avg_temp_duration / 4
try:
t = os.path.getmtime(self.last_avg_computation_file)
except OSError:
t = 0
if (time.time() - t) > avg_computation_period:
open(self.last_avg_computation_file, 'w').close()
temp_list = get_data_interval(self.__log_file, avg_temp_duration)
if temp_list:
avg_temp = sum(map(lambda x: x['cpu_temperature'], temp_list)) / len(temp_list)
if avg_temp > max_avg_temp:
self.logger.error("Average temperature over the last %s seconds "\
"reached threshold: %s degrees celsius (threshold is %s degrees "\
"celsius)" % (avg_temp_duration, avg_temp, max_avg_temp))
promise_success = False
else:
self.logger.error("Couldn't read temperature from log")
promise_success = False
if promise_success:
self.logger.info("Temperature OK")
def test(self):
"""
Called after sense() if the instance is still converging.
Returns success or failure based on sense results.
In this case, fail if the previous sensor result is negative.
"""
return self._test(result_count=1, failure_amount=1)
def anomaly(self):
"""
Called after sense() if the instance has finished converging.
Returns success or failure based on sense results.
Failure signals the instance has diverged.
In this case, fail if two out of the last three results are negative.
"""
return self._anomaly(result_count=3, failure_amount=2)
[buildout]
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../../stack/monitor/buildout.cfg
../../component/logrotate/buildout.cfg
parts +=
template
slapos-cookbook
slapos.toolbox-repository
develop =
${:parts-directory}/slapos.toolbox-repository
[slapos.toolbox-repository]
recipe = slapos.recipe.build:gitclone
repository = /srv/slapgrid/slappart72/srv/project/slapos.toolbox.git
branch = json-promise
git-executable = ${git:location}/bin/git
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
[download-base]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
[template-default]
<= download-base
[cpu-promise]
<= download-base
Tests for promise testing software release
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.promise_testing'
with open("README.md") as f:
long_description = f.read()
setup(
name=name,
version=version,
description="Test for SlapOS' promise-testing",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'slapos.cookbook',
],
zip_safe=True,
test_suite='test',
)
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import json
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, TestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
#TODO: use mock for temperature ?
class TestInstance(TestCase):
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({
'max_temp': 70,
'testing': True,
})}
@classmethod
def getInstanceSoftwareType(cls):
return "default"
def test_instance(self):
pass
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment