Skip to content

S
slapos.core
Commit f7eadb26 authored by Alain Takoudjou's avatar Alain Takoudjou
1 Browse files
Options

grid.promise: implement a new promise launcher in slapgrid 

python promises can define sense, test and anomaly method.
test method will be called by slapgrid or when no bang is needed for the promise anomaly method is
called when a promise failure required to bang the master, anomaly method can be optional but it's not the case for test method.

Slapgrid always run promises and save the result in .slapgrid/promise/result in a JSON format. The result will be used later by monitor
When a partition is correctly deployed, slapgrid will only run promise anomaly and will bang if there is an error and if the failed promise can bang

check promise anomaly when partition is upto date
parent 65c7a06d
Show whitespace changes
Inline Side-by-side
Showing with 2288 additions and 22 deletions
slapos/grid/promise/__init__.py 0 → 100644
View file @ f7eadb26
# -*- coding: utf-8 -*-
# vim: set et sts=2:
##############################################################################
#
# Copyright (c) 2018 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import sys
import logging
import time
import re
import json
import importlib
import traceback
import psutil
from multiprocessing import Process, Queue as MQueue
import Queue
from slapos.util import mkdir_p, chownDirectory
from slapos.grid.utils import dropPrivileges
from slapos.grid.promise import interface
from slapos.grid.promise.generic import (GenericPromise, PromiseQueueResult,
AnomalyResult, TestResult,
PROMISE_RESULT_FOLDER_NAME,
PROMISE_STATE_FOLDER_NAME)
from slapos.grid.promise.wrapper import WrapPromise
class PromiseError(Exception):
pass
class PromiseRunner(Process):
"""
Run a promise in a new Process
"""
def __init__(self, promise_instance, logger=None, allow_bang=True, uid=None,
gid=None, cwd=None, check_anomaly=False):
"""
Initialise Promise Runner
@param promise_instance: Promise instance from GenericPromise class
@param allow_bang: Bolean saying if bang should be called in case of
anomaly failure.
"""
Process.__init__(self)
self.promise = promise_instance
self.logger = logger
self.allow_bang = allow_bang
self.check_anomaly = check_anomaly
self.uid = uid
self.gid = gid
self.cwd = cwd
def run(self):
if self.uid and self.gid:
dropPrivileges(self.uid, self.gid, logger=self.logger)
if self.cwd is not None:
os.chdir(self.cwd)
try:
self.promise.run(self.check_anomaly, self.allow_bang)
except Exception, e:
if self.logger:
self.logger.error(str(e))
raise
class PromiseLauncher(object):
def __init__(self, config=None, logger=None, dry_run=False):
"""
Promise launcher will run promises
@param config_file: A file containing configurations
@param dry_run: Only run all promises without save the result
@param logger: Set the logger to use, if None a logger will be configured
to console.
@param config: A configuration dict to use. Values send here will
overwrite configs from `config_file`. Expected values in config are:
promise-timeout
Maximum promise execution time before timeout. Default: 20
partition-folder
Base path of the partition
promise-folder
Promises folder, all promises scripts will be imported from that folder
legacy-promise-folder
Legacy promises folder, where to find bash, shell and standard promises
log-folder
Folder where promises will write logs. Can be None
check-anomaly
Ask to check anomaly instead of test. Default: False
debug
Configure loggin in debug mode. Default: True
master-url
SlapOS Master service URL
partition-cert
Computer Partition Certificate file
partition-key
Computer Partition key file
partition-id
Computer Partition ID, ex: slappart13
computer-id
Computer ID, ex: COMP-1234
uid
User UID
gid
User GID
debug
If True, show Promise consumption and execution time information, etc
run-only-promise-list
A list of promise from plugins directory that will be executed
force
Set to True if force run promises without check their periodicity
"""
self.dry_run = dry_run
self.__config = {
'promise-timeout': 20,
'promise-folder': None,
'legacy-promise-folder': None,
'log-folder': None,
'partition-folder': None,
'debug': False,
'uid': None,
'gid': None,
'master-url': None,
'partition-cert': None,
'partition-key': None,
'partition-id': None,
'computer-id': None,
'check-anomaly': False,
'force': False,
'run-only-promise-list': None
}
if config is not None:
self.__config.update(config)
for key, value in self.__config.items():
setattr(self, key.replace('-', '_'), value or None)
if self.promise_folder is None:
raise ValueError("Promise folder is missing in configuration!")
if self.partition_folder is None:
raise ValueError("Partition folder is missing in configuration!")
if logger is None:
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.DEBUG if self.debug else logging.INFO)
if len(self.logger.handlers) == 0 or \
not isinstance(self.logger.handlers[0], logging.StreamHandler):
handler = logging.StreamHandler()
handler.setFormatter(
logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
)
self.logger.addHandler(handler)
else:
self.logger = logger
self.queue_result = MQueue()
self.bang_called = False
def _loadPromiseModule(self, promise_name):
"""Load a promise from promises directory."""
if re.match(r'[a-zA-Z_]', promise_name) is None:
self.logger.error("Promise plugin name %r is not valid" % promise_name)
promise_module = importlib.import_module(os.path.splitext(promise_name)[0])
if not hasattr(promise_module, "RunPromise"):
raise AttributeError("Class RunPromise not found in promise" \
"%s" % promise_name)
if not interface.IPromise.implementedBy(promise_module.RunPromise):
raise RuntimeError("RunPromise class in %s must implements 'IPromise'" \
" interface. zope_interface.implements(interface.IPromise) is" \
" missing ?" % promise_name)
if not issubclass(promise_module.RunPromise, GenericPromise):
raise RuntimeError("RunPromise class is not a subclass of" \
"GenericPromise class.")
return promise_module
def _getErrorPromiseResult(self, promise_instance, promise_name, message,
execution_time=0):
if self.check_anomaly:
result = AnomalyResult(problem=True, message=message)
else:
result = TestResult(problem=True, message=message)
return PromiseQueueResult(
item=result,
path=os.path.join(self.promise_folder, promise_name),
name=promise_name,
title=promise_instance.getTitle(),
execution_time=execution_time
)
def _savePromiseResult(self, result):
if not isinstance(result, PromiseQueueResult):
self.logger.error('Bad result: %s is not type of PromiseQueueResult...' % result)
return
promise_output_dir = os.path.join(
self.partition_folder,
PROMISE_RESULT_FOLDER_NAME
)
promise_output_file = os.path.join(
promise_output_dir,
"%s.status.json" % result.title
)
promise_tmp_file = '%s.tmp' % promise_output_file
if not os.path.exists(promise_output_dir):
mkdir_p(promise_output_dir)
with open(promise_tmp_file, "w") as outputfile:
json.dump(result.serialize(), outputfile)
os.rename(promise_tmp_file, promise_output_file)
def _launchPromise(self, promise_name, argument_dict, promise_module=None):
"""
Launch the promise and save the result if `self.save_method` is not None
If no save method is set, raise PromiseError in case of failure
"""
try:
if promise_module is None:
promise_instance = WrapPromise(argument_dict)
else:
promise_instance = promise_module.RunPromise(argument_dict)
if not self.force and not promise_instance.isPeriodicityMatch():
return False
promise_instance.setPromiseRunTimestamp()
except Exception:
# only print traceback to not prevent run other promises
self.logger.error(traceback.format_exc())
self.logger.warning("Promise %s skipped." % promise_name)
return True
promise_process = PromiseRunner(
promise_instance,
check_anomaly=self.check_anomaly,
allow_bang=not (self.bang_called or self.dry_run) and self.check_anomaly,
uid=self.uid,
gid=self.gid,
cwd=self.partition_folder,
logger=self.logger
)
self.logger.info("Checking promise %s..." % promise_name)
# set deamon to True, so promise process will be terminated if parent exit
promise_process.daemon = True
promise_process.start()
queue_item = None
sleep_time = 0.1
increment_limit = int(self.promise_timeout / sleep_time)
execution_time = self.promise_timeout
ps_profile = False
if self.debug:
try:
psutil_process = psutil.Process(promise_process.pid)
ps_profile = True
except psutil.NoSuchProcess:
# process is gone
pass
for current_increment in range(0, increment_limit):
if not promise_process.is_alive():
try:
queue_item = self.queue_result.get(True, 1)
except Queue.Empty:
# no result found in process result Queue
pass
else:
queue_item.execution_time = execution_time
break
if ps_profile:
try:
io_counter = psutil_process.io_counters()
self.logger.debug(
"[t=%ss] CPU: %s%%, MEM: %s MB (%s%%), DISK: %s Read - %s Write" % (
current_increment*sleep_time,
psutil_process.cpu_percent(),
psutil_process.memory_info().rss / float(2 ** 20),
round(psutil_process.memory_percent(), 4),
io_counter.read_count,
io_counter.write_count
)
)
except (psutil.AccessDenied, psutil.NoSuchProcess):
# defunct process will raise AccessDenied
pass
time.sleep(sleep_time)
execution_time = (current_increment + 1) * sleep_time
else:
promise_process.terminate()
promise_process.join() # wait for process to terminate
message = 'Promise timed out after %s seconds' % self.promise_timeout
queue_item = self._getErrorPromiseResult(
promise_instance,
promise_name=promise_name,
message=message,
execution_time=execution_time
)
if queue_item is None:
queue_item = self._getErrorPromiseResult(
promise_instance,
promise_name=promise_name,
message="No output returned by the promise",
execution_time=execution_time
)
if not self.dry_run:
self._savePromiseResult(queue_item)
if queue_item.item.hasFailed():
self.logger.error(queue_item.item.message)
if isinstance(queue_item.item, AnomalyResult) and self.check_anomaly:
# stop to bang as it was called
self.bang_called = True
if self.debug:
self.logger.debug("Finished promise %r in %s second(s)." % (
promise_name, execution_time))
return queue_item.item.hasFailed()
def run(self):
"""
Run all promises
"""
promise_list = []
promise_failed = False
base_config = {
'log-folder': self.log_folder,
'partition-folder': self.partition_folder,
'debug': self.debug,
'promise-timeout': self.promise_timeout,
'master-url': self.master_url,
'partition-cert': self.partition_cert,
'partition-key': self.partition_key,
'partition-id': self.partition_id,
'computer-id': self.computer_id,
'queue': self.queue_result,
}
if os.path.exists(self.promise_folder) and os.path.isdir(self.promise_folder):
# if there is no __init__ file, add it
init_file = os.path.join(self.promise_folder, '__init__.py')
if not os.path.exists(init_file):
with open(init_file, 'w') as f:
f.write("")
os.chmod(init_file, 0644)
if sys.path[0] != self.promise_folder:
sys.path[0:0] = [self.promise_folder]
promise_list = []
# load all promises so we can catch import errors before launch them
for promise_name in os.listdir(self.promise_folder):
if promise_name.startswith('__init__') or \
not promise_name.endswith('.py'):
continue
if self.run_only_promise_list is not None and not \
promise_name in self.run_only_promise_list:
continue
promise_list.append((promise_name,
self._loadPromiseModule(promise_name)))
for name, module in promise_list:
promise_path = os.path.join(self.promise_folder, name)
config = {
'path': promise_path,
'name': name
}
if module.__file__ != promise_path:
# cached module need to be updated
module = reload(module)
config.update(base_config)
if self._launchPromise(name, config, module) and not promise_failed:
promise_failed = True
if not self.run_only_promise_list and os.path.exists(self.legacy_promise_folder) \
and os.path.isdir(self.legacy_promise_folder):
# run legacy promise styles
for promise_name in os.listdir(self.legacy_promise_folder):
promise_path = os.path.join(self.legacy_promise_folder, promise_name)
if not os.path.isfile(promise_path) or \
not os.access(promise_path, os.X_OK):
self.logger.warning("Bad promise file at %r." % promise_path)
continue
config = {
'path': promise_path,
'name': promise_name
}
config.update(base_config)
# We will use promise wrapper to run this
if self._launchPromise(promise_name, config) and not promise_failed:
promise_failed = True
state_dir = os.path.join(self.partition_folder, PROMISE_STATE_FOLDER_NAME)
stat_info = os.stat(state_dir)
chownDirectory(state_dir, stat_info.st_uid, stat_info.st_gid)
if promise_failed:
raise PromiseError("Promise(s) has failed.")
slapos/grid/promise/generic.py 0 → 100644
View file @ f7eadb26
# -*- coding: utf-8 -*-
# vim: set et sts=2:
##############################################################################
#
# Copyright (c) 2018 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import logging
import re
import time
import random
import traceback
import slapos.slap
from slapos.util import mkdir_p
from abc import ABCMeta, abstractmethod
from datetime import datetime, timedelta
PROMISE_STATE_FOLDER_NAME = '.slapgrid/promise'
PROMISE_RESULT_FOLDER_NAME = '.slapgrid/promise/result'
PROMISE_LOG_FOLDER_NAME = '.slapgrid/promise/log'
class BaseResult(object):
def __init__(self, problem=False, message=None, date=None):
self.__problem = problem
self.__message = message
self.__date = date
if self.__date is None:
self.__date = datetime.utcnow()
def hasFailed(self):
return self.__problem
@property
def type(self):
return "Base Result"
@property
def message(self):
return self.__message
@property
def date(self):
return self.__date
class TestResult(BaseResult):
@property
def type(self):
return "Test Result"
class AnomalyResult(BaseResult):
@property
def type(self):
return "Anomaly Result"
class PromiseQueueResult(object):
def __init__(self, path, name, title, item, execution_time=0):
self.path = path
self.name = name
self.item = item
self.title = title
self.execution_time = execution_time
def serialize(self):
return {
'title': self.title,
'name': self.name,
'path': self.path,
'execution-time': self.execution_time,
'result': {
'type': self.item.type,
'failed': self.item.hasFailed(),
'date': self.item.date.strftime('%Y-%m-%dT%H:%M:%S'),
'message': self.item.message
}
}
class GenericPromise(object):
# Abstract class
__metaclass__ = ABCMeta
def __init__(self, config):
self.__config = config
self.__log_folder = self.__config.pop('log-folder', None)
self.__partition_folder = self.__config.pop('partition-folder', None)
self.__periodicity = self.__config.pop('periodicity', 2)
self.__debug = self.__config.pop('debug', True)
self.__name = self.__config.pop('name', None)
self.__promise_path = self.__config.pop('path', None)
self.__queue = self.__config.pop('queue', None)
self.__logger_buffer = None
self.__transaction_id = '%s-%s' % (int(time.time()), random.randint(100, 999))
self._validateConf()
self._configureLogger()
for key, value in config.items():
setattr(self, key.replace('-', '_'), value)
self.__timestamp_file = os.path.join(self.__partition_folder,
PROMISE_STATE_FOLDER_NAME,
self.__name)
def _configureLogger(self):
self.logger = logging.getLogger(self.__name)
for handler in self.logger.handlers:
self.logger.removeHandler(handler)
if self.__log_folder is None:
# configure logger with StringIO
import cStringIO
self.__logger_buffer = cStringIO.StringIO()
logger_handler = logging.StreamHandler(self.__logger_buffer)
self.__log_file = None
else:
self.__log_file = os.path.join(
self.__log_folder,
'%s.log' % self.__title
)
logger_handler = logging.FileHandler(self.__log_file)
self.logger.setLevel(logging.DEBUG if self.__debug else logging.INFO)
logger_handler.setFormatter(
fmt=logging.Formatter("%(asctime)s - %(levelname)s - " +
self.__transaction_id + " - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
)
self.logger.addHandler(logger_handler)
def _validateConf(self):
if self.__queue is None:
raise ValueError("Queue object is not set in configuration")
if self.__name is None:
raise ValueError("Monitor name is not set in configuration")
self.__title = os.path.splitext(self.__name)[0]
if self.__promise_path is None:
raise ValueError("Promise path is not set in configuration")
if self.__partition_folder is None:
raise ValueError("Monitor partition folder is not set in configuration")
def getConfig(self, key, default=None):
return self.__config.get(key, default)
def setConfig(self, key, value):
self.__config[key] = value
def getTitle(self):
return self.__title
def getName(self):
return self.__name
def getLogFile(self):
return self.__log_file
def getLogFolder(self):
return self.__log_folder
def getPartitionFolder(self):
return self.__partition_folder
def getPromiseFile(self):
return self.__promise_path
def setPeriodicity(self, minute):
if minute <= 0:
raise ValueError("Cannot set promise periodicity to a value less than 1")
self.__periodicity = minute
def getPeriodicity(self):
return self.__periodicity
def isPeriodicityMatch(self):
"""
Return True if promise should be run now, considering given the execution
periodicity in minutes
"""
if os.path.exists(self.__timestamp_file) and \
os.stat(self.__timestamp_file).st_size:
with open(self.__timestamp_file) as f:
try:
latest_timestamp = float(f.read())
current_timediff = (time.time() - latest_timestamp) / 60.0
if current_timediff >= self.__periodicity:
return True
self.logger.debug("Skip Promise %r. periodicity=%s, time_diff=%s" % (
self.__name, self.__periodicity, current_timediff))
except ValueError:
# if the file is broken, run the promise and regenerate it
return True
else:
return False
return True
def setPromiseRunTimestamp(self):
"""
Save the promise execution timestamp
"""
state_directory = os.path.dirname(self.__timestamp_file)
mkdir_p(state_directory)
with open(self.__timestamp_file, 'w') as f:
f.write(str(time.time()))
def __bang(self, message):
"""
Call bang if requested
"""
if self.__config.has_key('master-url') and \
self.__config.has_key('partition-id') and \
self.__config.has_key('computer-id'):
slap = slapos.slap.slap()
slap.initializeConnection(
self.__config['master-url'],
self.__config.get('partition-key'),
self.__config.get('partition-cert'),
)
computer_partition = slap.registerComputerPartition(
self.__config['computer-id'],
self.__config['partition-id'],
)
computer_partition.bang(message)
self.logger.info("Bang with message %r." % message)
def __getLogRegex(self):
return re.compile(r"(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})\s+\-?\s*(\w{4,7})\s+\-?\s+(\d+\-\d{3})\s+\-?\s*(.*)")
def __getResultFromString(self, result_string, only_failure=False):
line_list = result_string.split('\n')
result_list = []
line_part = ""
regex = self.__getLogRegex()
for line in line_list:
if not line:
continue
match = regex.match(line)
if match is not None:
if not only_failure or (only_failure and match.groups()[1] == 'ERROR'):
result_list.append({
'date': datetime.strptime(match.groups()[0], '%Y-%m-%d %H:%M:%S'),
'status': match.groups()[1],
'message': (match.groups()[3] + line_part).strip(),
})
line_part = ""
else:
line_part += '\n' + line
result_list
return [result_list]
def getLastPromiseResultList(self, latest_minute=0, result_count=1,
only_failure=False):
"""
Return the latest log result of the promise starting from the most recent
@param last_minute: the number of minutes in the past. If last_minute is
1, it will return the log of the latest minute execution. 0 => disabled
@param only_failure: only return the lines which contain failures.
@param result_count: maximum number of promise result to check, will not
exceed the `latest_minute`
@return Return a list of logs. The format is
[[{"date": "DATE", "status": "STATUS", "message": MESSAGE}, ...], ...]
"""
if self.__log_file is None:
if self.__logger_buffer is not None:
return self.__getResultFromString(self.__logger_buffer.getvalue(),
only_failure)
else:
return []
if not os.path.exists(self.__log_file):
return []
regex = self.__getLogRegex()
max_date_string = ""
if latest_minute > 0:
date = datetime.now() - timedelta(minutes=latest_minute)
max_date_string = date.strftime('%Y-%m-%d %H:%M:%S')
line_list = []
result_list = []
transaction_id = None
transaction_count = 0
with open(self.__log_file, 'r') as f:
offset = 0
f.seek(0, 2)
size = f.tell() * -1
line = line_part = ""
while offset > size:
offset -= 1
f.seek(offset, 2)
char = f.read(1)
if char != '\n':
line = char + line
if char == '\n' or offset == size:
# Add new line
if offset == -1:
continue
if line != "":
result = regex.match(line)
if result is not None:
if max_date_string and result.groups()[0] <= max_date_string:
break
if transaction_id != result.groups()[2]:
if transaction_id is not None:
# append new result
result_list.append(line_list)
line_list = []
transaction_count += 1
if transaction_count > result_count:
break
transaction_id = result.groups()[2]
if not only_failure or \
(only_failure and result.groups()[1] == 'ERROR'):
line_list.insert(0, {
'date': datetime.strptime(result.groups()[0],
'%Y-%m-%d %H:%M:%S'),
'status': result.groups()[1],
'message': (result.groups()[3] + line_part).strip(),
})
else:
line_part = '\n' + line + line_part
line = ""
continue
line = line_part = ""
if len(line_list):
result_list.append(line_list)
return result_list
def __readResultList(self, result_list):
failed = False
message = ""
for result in result_list:
if result['status'] == 'ERROR':
failed = True
message += "\n%s" % result['message']
return failed, message.strip()
def __checkPromiseResult(self, result_count=1, failure_amount=1,
latest_minute=0, is_anomaly=False):
"""
Test if the latest messages contain `failure_amount` failures.
@param result_count: maximum number of promise result to check, will not
exceed the `latest_minute`
@param latest_minute: test the result from now to the latest X minutes in
the past.
@param failure_amount: fail is this amount of failure is found in result
@param is_anomaly: Say if the result is an AnomalyResult of TestResult
"""
module = TestResult if not is_anomaly else AnomalyResult
latest_result_list = self.getLastPromiseResultList(
result_count=result_count,
latest_minute=latest_minute,
only_failure=False
)
result_size = len(latest_result_list)
if result_size == 0:
return module(problem=False, message="No result found!")
problem, message = self.__readResultList(latest_result_list[0])
if not problem:
# latest execution is OK
return module(problem=False, message=message)
i = 1
failure_found = 1
while i < result_size and failure_found < failure_amount:
for result in latest_result_list[i]:
if result['status'] == 'ERROR':
failure_found += 1
break
i += 1
if failure_found != failure_amount:
return module(problem=False, message=message)
return module(problem=True, message=message)
def _test(self, result_count=1, failure_amount=1, latest_minute=0):
"""
Default promise test method
"""
return self.__checkPromiseResult(
result_count=result_count,
failure_amount=failure_amount,
latest_minute=latest_minute,
is_anomaly=False
)
def _anomaly(self, result_count=1, failure_amount=1, latest_minute=0):
"""
Default anomaly check method
"""
return self.__checkPromiseResult(
result_count=result_count,
failure_amount=failure_amount,
latest_minute=latest_minute,
is_anomaly=True
)
@abstractmethod
def sense(self):
"""Run the promise code and log the result"""
def anomaly(self):
"""Called to detect if there is an anomaly which require to bang."""
return self._anomaly()
def test(self):
"""Test promise and say if problem is detected or not"""
return self._test()
def run(self, check_anomaly=False, can_bang=True):
"""
Method called to run the Promise
@param check_anomaly: Say if anomaly method should be called
@param can_bang: Set to True if bang can be called, this parameter should
be set to False if bang is already called by another promise.
"""
try:
self.sense()
except Exception, e:
# log the result
self.logger.error(str(e))
if check_anomaly:
# run sense, anomaly
try:
result = self.anomaly()
if result is None:
raise ValueError("Promise anomaly method returned 'None'")
except Exception, e:
result = AnomalyResult(problem=True, message=str(e))
else:
if isinstance(result, AnomalyResult) and result.hasFailed() and can_bang:
try:
self.__bang("Promise %s is failing" % self.__title)
except:
self.logger.warning(traceback.format_exc())
else:
# run sense, test
try:
result = self.test()
if result is None:
raise ValueError("Promise test method returned 'None'")
except Exception, e:
result = TestResult(problem=True, message=str(e))
if self.__logger_buffer is not None:
self.__logger_buffer.close()
# send the result of this promise
self.__queue.put(PromiseQueueResult(
path=self.__promise_path,
name=self.__name,
title=self.__title,
item=result
), True)
slapos/grid/promise/interface.py 0 → 100644
View file @ f7eadb26
# coding: utf-8
from zope.interface import Interface
class IPromise(Interface):
"""Base Promise interface."""
def __init__(config):
"""
@param config: Configurations needed to start the promise
"""
def anomaly(self):
"""
Called to detect if there is an anomaly.
@return AnomalyResult object
"""
def sense(self):
"""
Run the promise code and store the result
raise error, log error message, ... for failure
"""
def test(self):
"""
Test promise and say if problem is detected or not
@return TestResult object
"""
slapos/grid/promise/wrapper.py 0 → 100644
View file @ f7eadb26
# -*- coding: utf-8 -*-
# vim: set et sts=2:
##############################################################################
#
# Copyright (c) 2018 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import subprocess
import functools
import signal
import traceback
from zope import interface as zope_interface
from slapos.grid.promise import interface
from slapos.grid.promise.generic import GenericPromise
class WrapPromise(GenericPromise):
"""
A wrapper promise used to run old promises style and bash promises
"""
zope_interface.implements(interface.IPromise)
def __init__(self, config):
GenericPromise.__init__(self, config)
self.setPeriodicity(minute=2)
@staticmethod
def terminate(name, logger, process, signum, frame):
if signum in [signal.SIGINT, signal.SIGTERM] and process:
logger.info("Terminating promise process %r" % name)
try:
# make sure we kill the process on timeout
process.terminate()
except Exception:
logger.error(traceback.format_exc())
def sense(self):
promise_process = subprocess.Popen(
[self.getPromiseFile()],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=self.getPartitionFolder()
)
handler = functools.partial(self.terminate, self.getName(), self.logger,
promise_process)
signal.signal(signal.SIGINT, handler)
signal.signal(signal.SIGTERM, handler)
output, error = promise_process.communicate()
message = output or ""
if error:
  • Julien Muchembled @jm ·
    Owner

    How can 'error' be non-empty if Popen is called with stderr=subprocess.STDOUT ?

    (@bminusl will fix as part of py3 work)

    @nexedi Please learn 'subprocess' module seriously before using it. It's so rare to see it used correctly.

Please register or sign in to reply
message += "\n" + error
if promise_process.returncode != 0:
self.logger.error(message.strip())
else:
self.logger.info(message.strip())
def test(self):
# Fail if the latest promise result failed
return self._test(result_count=1, failure_amount=1)
def anomaly(self):
# Fail if 3 latest promise result failed, no bang
return self._test(result_count=3, failure_amount=3)
slapos/grid/slapgrid.py
View file @ f7eadb26
......@@ -60,8 +60,9 @@ from slapos.grid.svcbackend import (launchSupervisord,
createSupervisordConfiguration,
_getSupervisordConfigurationDirectory,
_getSupervisordSocketPath)
from slapos.grid.utils import (md5digest, dropPrivileges, SlapPopen, updateFile,
checkPromiseList, PromiseError)
from slapos.grid.utils import (md5digest, dropPrivileges, SlapPopen, updateFile)
from slapos.grid.promise import PromiseLauncher, PromiseError
from slapos.grid.promise.generic import PROMISE_LOG_FOLDER_NAME
from slapos.human import human2bytes
import slapos.slap
from netaddr import valid_ipv4, valid_ipv6
......@@ -617,22 +618,39 @@ stderr_logfile_backups=1
return SLAPGRID_FAIL
return SLAPGRID_SUCCESS
def _checkPromiseList(self, computer_partition):
self.logger.info("Checking promises...")
instance_path = os.path.join(self.instance_root, computer_partition.getId())
def _checkPromiseList(self, partition, force=True, check_anomaly=False):
instance_path = os.path.join(self.instance_root, partition.partition_id)
promise_log_path = os.path.join(instance_path, PROMISE_LOG_FOLDER_NAME)
mkdir_p(promise_log_path)
self.logger.info("Checking %s promises..." % partition.partition_id)
uid, gid = None, None
stat_info = os.stat(instance_path)
#stat sys call to get statistics informations
uid = stat_info.st_uid
gid = stat_info.st_gid
promise_dir = os.path.join(instance_path, 'etc', 'promise')
if not checkPromiseList(promise_dir, self.promise_timeout, uid=uid, gid=gid,
cwd=instance_path, logger=self.logger, profile=False,
raise_on_failure=True):
self.logger.info("No promise.")
promise_dir = os.path.join(instance_path, 'etc', 'plugin')
legacy_promise_dir = os.path.join(instance_path, 'etc', 'promise')
promise_config = {
'promise-folder': promise_dir,
'legacy-promise-folder': legacy_promise_dir,
'promise-timeout': self.promise_timeout,
'uid': uid,
'gid': gid,
'partition-folder': instance_path,
'log-folder': promise_log_path,
'force': force,
'check-anomaly': check_anomaly,
'master-url': partition.server_url,
'partition-cert': partition.cert_file,
'partition-key': partition.key_file,
'partition-id': partition.partition_id,
'computer-id': self.computer_id,
}
promise_checker = PromiseLauncher(config=promise_config, logger=self.logger)
return promise_checker.run()
def _endInstallationTransaction(self, computer_partition):
partition_id = computer_partition.getId()
......@@ -946,6 +964,7 @@ stderr_logfile_backups=1
# Try to process it anyway, it may need to be deleted.
software_path = None
computer_partition_state = computer_partition.getState()
periodicity = self.maximum_periodicity
if software_path:
periodicity_path = os.path.join(software_path, 'periodicity')
......@@ -996,7 +1015,14 @@ stderr_logfile_backups=1
# Check periodicity, i.e if periodicity is one day, partition
# should be processed at least every day.
if int(time.time()) <= (last_runtime + periodicity) or periodicity < 0:
self.logger.debug('Partition already up-to-date, skipping.')
# check promises anomaly
if computer_partition_state == COMPUTER_PARTITION_STARTED_STATE:
self.logger.debug('Partition already up-to-date.')
self._checkPromiseList(local_partition,
check_anomaly=True,
force=False)
else:
self.logger.debug('Partition already up-to-date. skipping.')
return
else:
# Periodicity forced processing this partition. Removing
......@@ -1028,8 +1054,6 @@ stderr_logfile_backups=1
self.logger.info(' Software path: %s' % software_path)
self.logger.info(' Instance path: %s' % instance_path)
computer_partition_state = computer_partition.getState()
# XXX this line breaks 37 tests
# self.logger.info(' Instance type: %s' % computer_partition.getType())
self.logger.info(' Instance status: %s' % computer_partition_state)
......@@ -1046,7 +1070,7 @@ stderr_logfile_backups=1
if self.firewall_conf:
self._setupComputerPartitionFirewall(computer_partition,
partition_ip_list)
self._checkPromiseList(computer_partition)
self._checkPromiseList(local_partition)
computer_partition.started()
self._endInstallationTransaction(computer_partition)
elif computer_partition_state == COMPUTER_PARTITION_STOPPED_STATE:
......@@ -1085,7 +1109,14 @@ stderr_logfile_backups=1
with open(error_output_file, 'w') as error_file:
# Write error message in a log file assible to computer partition user
error_file.write(str(e))
raise
if not isinstance(e, PromiseError) and \
computer_partition_state == COMPUTER_PARTITION_STARTED_STATE:
try:
self._checkPromiseList(local_partition)
except PromiseError:
# updating promises state, no need to raise here
pass
raise e
else:
self.logger.removeHandler(partition_file_handler)
if os.path.exists(error_output_file):
......
slapos/grid/utils.py
View file @ f7eadb26
......@@ -192,7 +192,7 @@ def dropPrivileges(uid, gid, logger):
if uid == 0 or gid == 0:
raise OSError('Dropping privileges to uid = %r or '
'gid = %r is too dangerous' % (uid, gid))
if current_uid or current_gid:
if (current_uid or current_gid):
logger.debug('Running as uid = %r, gid = %r, dropping '
'not needed and not possible' % (current_uid, current_gid))
return
......
slapos/tests/promise.py 0 → 100644
View file @ f7eadb26
##############################################################################
#
# Copyright (c) 2018 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os, shutil
import tempfile
import unittest
import sys
import time
import json
import random
from datetime import datetime, timedelta
import Queue
from zope import interface as zope_interface
from slapos.grid.promise import interface, PromiseLauncher, PromiseError
from slapos.grid.promise.generic import (GenericPromise, TestResult, AnomalyResult,
PromiseQueueResult, PROMISE_STATE_FOLDER_NAME,
PROMISE_RESULT_FOLDER_NAME)
class TestSlapOSPromiseMixin(unittest.TestCase):
def setUp(self):
self.partition_dir = tempfile.mkdtemp()
self.plugin_dir = os.path.join(self.partition_dir, 'plugins')
self.legacy_promise_dir = os.path.join(self.partition_dir, 'promise')
self.log_dir = os.path.join(self.partition_dir, 'log')
os.mkdir(self.plugin_dir)
os.mkdir(self.log_dir)
os.makedirs('%s/.slapgrid/promise' % self.partition_dir)
os.mkdir(self.legacy_promise_dir)
self.partition_id = "slappart0"
self.computer_id = "COMP-1234"
def writeInit(self):
with open(os.path.join(self.plugin_dir, '__init__'), 'w') as f:
f.write('')
os.chmod(os.path.join(self.plugin_dir, '__init__'), 0644)
if sys.path[0] != self.plugin_dir:
sys.path[0:0] = [self.plugin_dir]
def tearDown(self):
if os.path.exists(self.partition_dir):
shutil.rmtree(self.partition_dir)
if sys.path[0] == self.plugin_dir:
del sys.path[0]
def configureLauncher(self, save_method=None, timeout=0.5, master_url="", debug=False,
run_list=[], uid=None, gid=None, enable_anomaly=False, force=False,
logdir=True, dry_run=False):
parameter_dict = {
'promise-timeout': timeout,
'promise-folder': self.plugin_dir,
'legacy-promise-folder': self.legacy_promise_dir,
'log-folder': self.log_dir if logdir else None,
'partition-folder': self.partition_dir,
'master-url': master_url,
'partition-cert': "",
'partition-key': "",
'partition-id': self.partition_id,
'computer-id': self.computer_id,
'debug': debug,
'check-anomaly': enable_anomaly,
'force': force,
'run-only-promise-list': run_list,
'uid': uid,
'gid': gid
}
self.launcher = PromiseLauncher(
config=parameter_dict,
logger=None,
dry_run=dry_run
)
if save_method:
self.launcher._savePromiseResult = save_method
def writeFile(self, path, content, mode=0644):
with open(path, 'w') as f:
f.write(content)
os.chmod(path, mode)
def generatePromiseScript(self, name, success=True, failure_count=1, content="",
periodicity=0.03):
promise_content = """from zope import interface as zope_interface
from slapos.grid.promise import interface
from slapos.grid.promise import GenericPromise
class RunPromise(GenericPromise):
zope_interface.implements(interface.IPromise)
def __init__(self, config):
GenericPromise.__init__(self, config)
self.setPeriodicity(minute=%(periodicity)s)
def sense(self):
%(content)s
if not %(success)s:
self.logger.error("failed")
else:
self.logger.info("success")
def anomaly(self):
return self._anomaly(latest_minute=%(periodicity)s, failure_amount=%(failure_amount)s)
def test(self):
return self._test(latest_minute=%(periodicity)s, failure_amount=%(failure_amount)s)
""" % {'success': success, 'content': content, 'failure_amount': failure_count,
'periodicity': periodicity}
with open(os.path.join(self.plugin_dir, name), 'w') as f:
f.write(promise_content)
class TestSlapOSPromiseLauncher(TestSlapOSPromiseMixin):
def test_promise_match_interface(self):
promise_name = 'my_promise.py'
self.configureLauncher()
self.generatePromiseScript(promise_name)
self.writeInit()
promise_module = self.launcher._loadPromiseModule(promise_name)
def test_promise_match_interface_bad_name(self):
promise_name = 'my_promise_no_py'
self.configureLauncher()
self.generatePromiseScript(promise_name)
self.writeInit()
with self.assertRaises(ImportError):
promise_module = self.launcher._loadPromiseModule(promise_name)
def test_promise_match_interface_no_implement(self):
promise_name = 'my_promise_noimplement.py'
promise_content = """from slapos.grid.promise import GenericPromise
class RunPromise(GenericPromise):
def __init__(self, config):
GenericPromise.__init__(self, config)
def sense(self):
pass
"""
promise_path = os.path.join(self.plugin_dir, promise_name)
self.configureLauncher()
self.writeInit()
self.writeFile(promise_path, promise_content)
with self.assertRaises(RuntimeError):
promise_module = self.launcher._loadPromiseModule(promise_name)
def test_promise_match_interface_no_generic(self):
promise_name = 'my_promise_nogeneric.py'
promise_content = """from zope import interface as zope_interface
from slapos.grid.promise import interface
class RunPromise(object):
zope_interface.implements(interface.IPromise)
def __init__(self, config):
pass
def sense(self):
pass
"""
promise_path = os.path.join(self.plugin_dir, promise_name)
self.configureLauncher()
self.writeInit()
self.writeFile(promise_path, promise_content)
with self.assertRaises(RuntimeError):
promise_module = self.launcher._loadPromiseModule(promise_name)
def test_promise_match_interface_no_sense(self):
promise_name = 'my_promise_nosense.py'
promise_content = """from zope import interface as zope_interface
from slapos.grid.promise import interface
from slapos.grid.promise import GenericPromise
class RunPromise(GenericPromise):
zope_interface.implements(interface.IPromise)
def __init__(self, config):
pass
def noSenseMethod(self):
pass
"""
promise_path = os.path.join(self.plugin_dir, promise_name)
self.configureLauncher()
self.writeInit()
self.writeFile(promise_path, promise_content)
with self.assertRaises(TypeError):
promise_module = self.launcher._loadPromiseModule(promise_name)
promise = promise_module.RunPromise({})
def test_runpromise(self):
promise_name = 'my_promise.py'
self.configureLauncher()
self.generatePromiseScript(promise_name, success=True)
state_folder = os.path.join(self.partition_dir, PROMISE_STATE_FOLDER_NAME)
# run promise will not fail
self.launcher.run()
self.assertTrue(os.path.exists(state_folder))
self.assertTrue(os.path.exists(os.path.join(self.log_dir, 'my_promise.log')))
expected_result = """{
"result": {
"failed": false,
"message": "success",
"type": "Test Result"
},
"path": "%s/my_promise.py",
"name": "my_promise.py",
"execution-time": 0.1,
"title": "my_promise"
}""" % self.plugin_dir
state_file = os.path.join(self.partition_dir, PROMISE_RESULT_FOLDER_NAME, 'my_promise.status.json')
self.assertTrue(os.path.exists(state_file))
with open(state_file) as f:
result_dict = json.loads(f.read())
result_dict['result'].pop('date')
self.assertEquals(json.loads(expected_result), result_dict)
def test_runpromise_multiple(self):
promise_name = 'my_promise.py'
second_name = 'my_second_promise.py'
self.configureLauncher()
self.generatePromiseScript(promise_name, success=True)
self.generatePromiseScript(second_name, success=True)
state_folder = os.path.join(self.partition_dir, PROMISE_STATE_FOLDER_NAME)
# run promise will not fail
self.launcher.run()
self.assertTrue(os.path.exists(state_folder))
self.assertTrue(os.path.exists(os.path.join(self.log_dir, 'my_promise.log')))
expected_result = """{
"result": {
"failed": false,
"message": "success",
"type": "Test Result"
},
"path": "%(promise_dir)s/%(name)s.py",
"name": "%(name)s.py",
"execution-time": 0.1,
"title": "%(name)s"
}"""
# first promise
state_file = os.path.join(self.partition_dir, PROMISE_RESULT_FOLDER_NAME, 'my_promise.status.json')
self.assertTrue(os.path.exists(state_file))
with open(state_file) as f:
result_dict = json.loads(f.read())
result_dict['result'].pop('date')
expected_dict = expected_result % {'promise_dir': self.plugin_dir, 'name': 'my_promise'}
self.assertEquals(json.loads(expected_dict), result_dict)
# second promise
state_file = os.path.join(self.partition_dir, PROMISE_RESULT_FOLDER_NAME, 'my_second_promise.status.json')
self.assertTrue(os.path.exists(state_file))
with open(state_file) as f:
result_dict = json.loads(f.read())
result_dict['result'].pop('date')
expected_dict = expected_result % {'promise_dir': self.plugin_dir, 'name': 'my_second_promise'}
self.assertEquals(json.loads(expected_dict), result_dict)
def test_runpromise_no_logdir(self):
promise_name = 'my_promise.py'
# no promise log output dir
self.configureLauncher(logdir=False)
self.generatePromiseScript(promise_name, success=True)
state_file = os.path.join(self.partition_dir, PROMISE_STATE_FOLDER_NAME)
# run promise will not fail
self.launcher.run()
self.assertTrue(os.path.exists(state_file))
self.assertFalse(os.path.exists(os.path.join(self.log_dir, 'my_promise.log')))
def test_runpromise_savemethod(self):
promise_name = 'my_promise.py'
def test_method(result):
self.assertTrue(isinstance(result, PromiseQueueResult))
self.assertTrue(isinstance(result.item, TestResult))
self.assertTrue(result.execution_time != 0)
self.assertEquals(result.title, 'my_promise')
self.assertEquals(result.name, promise_name)
self.assertEquals(result.path, os.path.join(self.plugin_dir, promise_name))
self.assertEquals(result.item.message, "success")
self.assertEquals(result.item.hasFailed(), False)
self.assertTrue(isinstance(result.item.date, datetime))
self.configureLauncher(save_method=test_method)
self.generatePromiseScript(promise_name, success=True)
state_file = os.path.join(self.partition_dir, PROMISE_STATE_FOLDER_NAME)
# run promise will not fail
self.launcher.run()
self.assertTrue(os.path.exists(state_file))
self.assertTrue(os.path.exists(os.path.join(self.log_dir, 'my_promise.log')))
def test_runpromise_savemethod_no_logdir(self):
promise_name = 'my_promise.py'
def test_method(result):
self.assertTrue(isinstance(result, PromiseQueueResult))
self.assertTrue(isinstance(result.item, TestResult))
self.assertTrue(result.execution_time != 0)
self.assertEquals(result.title, 'my_promise')
self.assertEquals(result.name, promise_name)
self.assertEquals(result.path, os.path.join(self.plugin_dir, promise_name))
self.assertEquals(result.item.message, "success")
self.assertEquals(result.item.hasFailed(), False)
self.assertTrue(isinstance(result.item.date, datetime))
# no promise log output dir
self.configureLauncher(logdir=False, save_method=test_method)
self.generatePromiseScript(promise_name, success=True)
state_file = os.path.join(self.partition_dir, PROMISE_STATE_FOLDER_NAME)
# run promise will not fail
self.launcher.run()
self.assertTrue(os.path.exists(state_file))
self.assertFalse(os.path.exists(os.path.join(self.log_dir, 'my_promise.log')))
def test_runpromise_savemethod_anomaly(self):
promise_name = 'my_promise.py'
def test_method(result):
self.assertTrue(isinstance(result, PromiseQueueResult))
self.assertTrue(isinstance(result.item, AnomalyResult))
self.assertTrue(result.execution_time != 0)
self.assertEquals(result.title, 'my_promise')
self.assertEquals(result.name, promise_name)
self.assertEquals(result.path, os.path.join(self.plugin_dir, promise_name))
self.assertEquals(result.item.message, "success")
self.assertEquals(result.item.hasFailed(), False)
self.assertTrue(isinstance(result.item.date, datetime))
self.configureLauncher(save_method=test_method, enable_anomaly=True)
self.generatePromiseScript(promise_name, success=True)
state_file = os.path.join(self.partition_dir, PROMISE_STATE_FOLDER_NAME)
# run promise will not fail
self.launcher.run()
self.assertTrue(os.path.exists(state_file))
self.assertTrue(os.path.exists(os.path.join(self.log_dir, 'my_promise.log')))
def test_runpromise_savemethod_multiple(self):
promise_name = 'my_promise.py'
promise_failed = 'my_failed_promise.py'
self.counter = 0
def test_method(result):
self.assertTrue(isinstance(result, PromiseQueueResult))
self.assertTrue(isinstance(result.item, TestResult))
self.assertTrue(result.name in [promise_failed, promise_name])
if result.name == promise_failed:
self.assertEquals(result.item.hasFailed(), True)
self.assertEquals(result.item.message, "failed")
else:
self.assertEquals(result.item.hasFailed(), False)
self.assertEquals(result.item.message, "success")
self.counter += 1
self.configureLauncher(save_method=test_method)
self.generatePromiseScript(promise_name, success=True)
self.generatePromiseScript(promise_failed, success=False)
state_file = os.path.join(self.partition_dir, PROMISE_STATE_FOLDER_NAME)
with self.assertRaises(PromiseError):
self.launcher.run()
self.assertEquals(self.counter, 2)
self.assertTrue(os.path.exists(state_file))
self.assertTrue(os.path.exists(os.path.join(self.log_dir, 'my_promise.log')))
self.assertTrue(os.path.exists(os.path.join(self.log_dir, 'my_failed_promise.log')))
def test_runpromise_savemethod_multiple_success(self):
first_promise = 'my_first_promise.py'
second_promise = 'my_second_promise.py'
third_promise = 'my_third_promise.py'
self.counter = 0
def test_method(result):
self.assertTrue(isinstance(result, PromiseQueueResult))
self.assertTrue(isinstance(result.item, TestResult))
self.assertTrue(result.name in [first_promise, second_promise, third_promise])
self.assertEquals(result.item.hasFailed(), False)
self.assertEquals(result.item.message, "success")
self.counter += 1
self.configureLauncher(save_method=test_method)
self.generatePromiseScript(first_promise, success=True)
self.generatePromiseScript(second_promise, success=True)
self.generatePromiseScript(third_promise, success=True)
state_file = os.path.join(self.partition_dir, PROMISE_STATE_FOLDER_NAME)
# run promise will not fail
self.launcher.run()
self.assertEquals(self.counter, 3)
self.assertTrue(os.path.exists(state_file))
self.assertTrue(os.path.exists(os.path.join(self.log_dir, 'my_first_promise.log')))
self.assertTrue(os.path.exists(os.path.join(self.log_dir, 'my_second_promise.log')))
self.assertTrue(os.path.exists(os.path.join(self.log_dir, 'my_third_promise.log')))
def test_runpromise_fail_and_success(self):
first_promise = 'my_first_promise.py'
second_promise = 'my_second_promise.py'
self.configureLauncher()
self.generatePromiseScript(first_promise, success=True)
self.generatePromiseScript(second_promise, success=False)
# run promise will fail when promise fail (usefull for slapgrid)
with self.assertRaises(PromiseError) as exc:
self.launcher.run()
self.assertEquals(exc.exception.message, 'Promise(s) has failed.')
if "my_second_promise" in sys.modules:
# force to reload the module without rerun python
os.system('rm %s/*.pyc' % self.plugin_dir)
del sys.modules["my_second_promise"]
self.generatePromiseScript(second_promise, success=True)
# wait next periodicity
time.sleep(2)
self.launcher.run()
log_file = os.path.join(self.log_dir, 'my_second_promise.log')
self.assertTrue(os.path.exists(log_file))
with open(log_file) as f:
line = f.readline()
self.assertTrue('failed' in line, line)
line = f.readline()
self.assertTrue('success' in line, line)
def test_runpromise_with_periodicity(self):
first_promise = 'my_first_promise.py'
second_promise = 'my_second_promise.py'
self.counter = 0
def test_method_first(result):
self.assertTrue(result.name in [first_promise, second_promise])
self.assertEquals(result.item.hasFailed(), False)
self.assertEquals(result.item.message, "success")
self.counter += 1
def test_method_one(result):
self.counter += 1
self.assertEquals(result.name, first_promise)
self.assertEquals(result.item.hasFailed(), False)
self.assertEquals(result.item.message, "success")
self.configureLauncher(save_method=test_method_first)
# ~2 seconds
self.generatePromiseScript(first_promise, success=True, periodicity=0.03)
# ~3 seconds
self.generatePromiseScript(second_promise, success=True, periodicity=0.05)
self.launcher.run()
self.assertEquals(self.counter, 2)
self.configureLauncher(save_method=test_method_one)
time.sleep(2)
self.counter = 0
self.launcher.run() # only my_first_promise will run
self.assertEquals(self.counter, 1)
time.sleep(3)
self.counter = 0
self.configureLauncher(save_method=test_method_first)
self.launcher.run()
self.assertEquals(self.counter, 2)
def test_runpromise_with_periodicity_same(self):
first_promise = 'my_first_promise.py'
second_promise = 'my_second_promise.py'
self.counter = 0
def test_method(result):
self.assertTrue(result.name in [first_promise, second_promise])
self.assertEquals(result.item.hasFailed(), False)
self.assertEquals(result.item.message, "success")
self.counter += 1
self.configureLauncher(save_method=test_method)
# ~2 seconds
self.generatePromiseScript(first_promise, success=True, periodicity=0.03)
self.generatePromiseScript(second_promise, success=True, periodicity=0.03)
self.launcher.run()
self.assertEquals(self.counter, 2)
self.configureLauncher(save_method=test_method)
time.sleep(1)
self.counter = 0
self.launcher.run() # run nothing
self.assertEquals(self.counter, 0)
time.sleep(1)
self.counter = 0
self.configureLauncher(save_method=test_method)
self.launcher.run()
self.assertEquals(self.counter, 2)
def test_runpromise_force(self):
first_promise = 'my_first_promise.py'
second_promise = 'my_second_promise.py'
self.counter = 0
def test_method(result):
self.assertTrue(result.name in [first_promise, second_promise])
self.assertEquals(result.item.hasFailed(), False)
self.assertEquals(result.item.message, "success")
self.counter += 1
self.configureLauncher(save_method=test_method)
# ~2 seconds
self.generatePromiseScript(first_promise, success=True, periodicity=0.03)
self.generatePromiseScript(second_promise, success=True, periodicity=0.03)
self.launcher.run()
self.assertEquals(self.counter, 2)
self.configureLauncher(save_method=test_method)
time.sleep(1)
self.counter = 0
self.launcher.run() # run nothing
self.assertEquals(self.counter, 0)
self.configureLauncher(save_method=test_method, force=True)
self.counter = 0
self.launcher.run() # will run all as force is True
self.assertEquals(self.counter, 2)
self.configureLauncher(save_method=test_method)
time.sleep(1)
self.counter = 0
self.launcher.run() # run nothing
self.assertEquals(self.counter, 0)
time.sleep(1)
self.counter = 0
self.configureLauncher(save_method=test_method)
self.launcher.run() # after 2 seconds will run all
self.assertEquals(self.counter, 2)
def test_runpromise_wrapped(self):
promise_name = "my_bash_promise"
promise_path = os.path.join(self.legacy_promise_dir, promise_name)
self.called = False
with open(promise_path, 'w') as f:
f.write("""#!/bin/bash
echo "success"
""")
os.chmod(promise_path, 0744)
def test_method(result):
self.called = True
self.assertTrue(isinstance(result, PromiseQueueResult))
self.assertTrue(isinstance(result.item, TestResult))
self.assertTrue(result.execution_time != 0)
self.assertEquals(result.title, promise_name)
self.assertEquals(result.name, promise_name)
self.assertEquals(result.path, os.path.join(self.legacy_promise_dir, promise_name))
self.assertEquals(result.item.message, "success")
self.assertEquals(result.item.hasFailed(), False)
self.assertTrue(isinstance(result.item.date, datetime))
self.configureLauncher(save_method=test_method)
state_file = os.path.join(self.partition_dir, PROMISE_STATE_FOLDER_NAME)
self.launcher.run()
self.assertTrue(self.called)
self.assertTrue(os.path.exists(state_file))
def test_runpromise_wrapped_failed(self):
promise_name = "my_bash_promise"
promise_path = os.path.join(self.legacy_promise_dir, promise_name)
with open(promise_path, 'w') as f:
f.write("""#!/bin/bash
echo "This promise failed"
exit 1
""")
os.chmod(promise_path, 0744)
self.configureLauncher()
state_file = os.path.join(self.partition_dir, PROMISE_STATE_FOLDER_NAME)
with self.assertRaises(PromiseError) as exc:
self.launcher.run()
self.assertEquals(exc.exception.message, 'Promise(s) has failed.')
def test_runpromise_wrapped_mixed(self):
self.called = 0
result_dict = {"my_bash_promise": "", "my_bash_promise2": "", "first_promise.py": "", "second_promise.py": ""}
def test_method(result):
self.called += 1
result_dict.pop(result.name)
if result.title == "first_promise" or result.title == "second_promise":
self.assertEquals(result.item.message, "success")
if result.title == "my_bash_promise":
self.assertEquals(result.item.message, "promise 1 succeeded")
if result.title == "my_bash_promise2":
self.assertEquals(result.item.message, "promise 2 succeeded")
self.assertEquals(result.item.hasFailed(), False)
promise_name = "my_bash_promise"
promise_path = os.path.join(self.legacy_promise_dir, promise_name)
promise_name2 = "my_bash_promise2"
promise_path2 = os.path.join(self.legacy_promise_dir, promise_name2)
with open(promise_path, 'w') as f:
f.write("""#!/bin/bash
echo "promise 1 succeeded"
exit 0
""")
os.chmod(promise_path, 0744)
with open(promise_path2, 'w') as f:
f.write("""#!/bin/bash
echo "promise 2 succeeded"
exit 0
""")
os.chmod(promise_path2, 0744)
self.generatePromiseScript("first_promise.py", success=True)
self.generatePromiseScript("second_promise.py", success=True)
self.configureLauncher(save_method=test_method)
self.launcher.run()
self.assertEquals(self.called, 4)
def test_runpromise_run_only(self):
first_promise = 'my_first_promise.py'
second_promise = 'my_second_promise.py'
third_promise = 'my_third_promise.py'
self.counter = 0
self.check_list = [first_promise, second_promise, third_promise]
def test_method(result):
self.assertTrue(result.name in self.check_list)
self.assertEquals(result.item.hasFailed(), False)
self.assertEquals(result.item.message, "success")
self.counter += 1
self.configureLauncher(save_method=test_method)
self.generatePromiseScript(first_promise, success=True)
self.generatePromiseScript(second_promise, success=True)
self.generatePromiseScript(third_promise, success=True)
# run promise will not fail
self.launcher.run()
self.assertEquals(self.counter, 3)
self.counter = 0
self.check_list = [second_promise]
self.configureLauncher(save_method=test_method, run_list=[second_promise], force=True)
time.sleep(1)
self.launcher.run()
self.assertEquals(self.counter, 1)
def test_runpromise_run_only_multiple(self):
first_promise = 'my_first_promise.py'
second_promise = 'my_second_promise.py'
third_promise = 'my_third_promise.py'
self.counter = 0
self.check_list = [first_promise, second_promise, third_promise]
def test_method(result):
self.assertTrue(result.name in self.check_list)
self.assertEquals(result.item.hasFailed(), False)
self.assertEquals(result.item.message, "success")
self.counter += 1
self.configureLauncher(save_method=test_method)
self.generatePromiseScript(first_promise, success=True)
self.generatePromiseScript(second_promise, success=True)
self.generatePromiseScript(third_promise, success=True)
# run promise will not fail
self.launcher.run()
self.assertEquals(self.counter, 3)
self.counter = 0
self.check_list = [third_promise, second_promise]
self.configureLauncher(save_method=test_method, run_list=self.check_list, force=True)
time.sleep(1)
self.launcher.run()
self.assertEquals(self.counter, 2)
def test_runpromise_will_timeout(self):
self.called = False
promise_name = 'my_promise.py'
def test_method(result):
self.called = True
self.assertTrue(isinstance(result, PromiseQueueResult))
self.assertTrue(isinstance(result.item, AnomalyResult))
self.assertTrue(result.execution_time >= 1)
self.assertEquals(result.title, 'my_promise')
self.assertEquals(result.name, promise_name)
self.assertTrue("Promise timed out after" in result.item.message)
self.assertEquals(result.item.hasFailed(), True)
self.configureLauncher(save_method=test_method, enable_anomaly=True, timeout=1)
self.generatePromiseScript(promise_name, success=True, content="""import time
time.sleep(5)""")
# run promise will timeout
with self.assertRaises(PromiseError):
self.launcher.run()
self.assertTrue(self.called)
class TestSlapOSGenericPromise(TestSlapOSPromiseMixin):
def initialisePromise(self, promise_content="", success=True, timeout=60):
self.promise_name = 'my_promise.py'
self.promise_path = os.path.join(self.plugin_dir, self.promise_name)
self.configureLauncher()
self.generatePromiseScript(self.promise_name, periodicity=1, content=promise_content, success=success)
self.writeInit()
self.queue = Queue.Queue()
self.promise_config = {
'log-folder': self.log_dir,
'partition-folder': self.partition_dir,
'promise-timeout': timeout,
'debug': False,
'slapgrid-mode': False,
'check-anomaly': True,
'master-url': "https://master.url.com",
'partition-cert': '',
'partition-key': '',
'partition-id': self.partition_id,
'computer-id': self.computer_id,
'queue': self.queue,
'path': self.promise_path,
'name': self.promise_name
}
def test_create_simple_promise(self):
self.initialisePromise()
promise_module = self.launcher._loadPromiseModule(self.promise_name)
reload(promise_module)
promise = promise_module.RunPromise(self.promise_config)
self.assertEquals(promise.getPeriodicity(), 1)
self.assertEquals(promise.getName(), self.promise_name)
self.assertEquals(promise.getTitle(), 'my_promise')
self.assertEquals(promise.getPartitionFolder(), self.partition_dir)
self.assertEquals(promise.getPromiseFile(), self.promise_path)
self.assertEquals(promise.getLogFolder(), self.log_dir)
self.assertEquals(promise.getLogFile(), os.path.join(self.log_dir, 'my_promise.log'))
promise.setPeriodicity(2)
self.assertEquals(promise.getPeriodicity(), 2)
with self.assertRaises(ValueError):
promise.setPeriodicity(0)
promise.run(check_anomaly=True)
result = self.queue.get(True, 1)
self.assertTrue(isinstance(result, PromiseQueueResult))
self.assertTrue(isinstance(result.item, AnomalyResult))
self.assertEquals(result.title, 'my_promise')
self.assertEquals(result.name, self.promise_name)
self.assertEquals(result.path, os.path.join(self.plugin_dir, self.promise_name))
self.assertEquals(result.item.message, "success")
self.assertEquals(result.item.hasFailed(), False)
self.assertTrue(isinstance(result.item.date, datetime))
def test_promise_anomaly_disabled(self):
self.initialisePromise()
promise_module = self.launcher._loadPromiseModule(self.promise_name)
reload(promise_module)
promise = promise_module.RunPromise(self.promise_config)
# disable anomaly call, enable test call
promise.setConfig("check-anomaly", False)
promise.run()
result = self.queue.get(True, 1)
self.assertTrue(isinstance(result, PromiseQueueResult))
self.assertTrue(isinstance(result.item, TestResult))
self.assertEquals(result.title, 'my_promise')
self.assertEquals(result.name, self.promise_name)
self.assertEquals(result.path, os.path.join(self.plugin_dir, self.promise_name))
self.assertEquals(result.item.message, "success")
self.assertEquals(result.item.hasFailed(), False)
self.assertTrue(isinstance(result.item.date, datetime))
def test_promise_with_raise(self):
promise_content = "raise ValueError('Bad Promise raised')"
self.initialisePromise(promise_content)
promise_module = self.launcher._loadPromiseModule(self.promise_name)
reload(promise_module)
promise = promise_module.RunPromise(self.promise_config)
# no raise
promise.run()
result = self.queue.get(True, 1)
self.assertEquals(result.title, 'my_promise')
self.assertEquals(result.name, self.promise_name)
self.assertEquals(result.item.message, "Bad Promise raised")
self.assertEquals(result.item.hasFailed(), True)
def test_promise_no_return(self):
promise_content = "return"
self.initialisePromise(promise_content)
promise_module = self.launcher._loadPromiseModule(self.promise_name)
reload(promise_module)
promise = promise_module.RunPromise(self.promise_config)
# no raise
promise.run()
result = self.queue.get(True, 1)
self.assertEquals(result.title, 'my_promise')
self.assertEquals(result.name, self.promise_name)
self.assertEquals(result.item.message, "No result found!")
self.assertEquals(result.item.hasFailed(), False)
def test_promise_resultfromlog(self):
promise_content = "self.logger.info('Promise is running...')"
self.initialisePromise(promise_content)
promise_module = self.launcher._loadPromiseModule(self.promise_name)
reload(promise_module)
promise = promise_module.RunPromise(self.promise_config)
date = datetime.now()
promise.sense()
# get all messages during the latest minute
latest_message_list = promise.getLastPromiseResultList(result_count=1)
date = datetime.strptime(date.strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')
self.assertEquals(len(latest_message_list), 1)
self.assertEquals(
latest_message_list[0][0],
{'date': date, 'status': 'INFO', 'message': 'Promise is running...'})
self.assertEquals(
latest_message_list[0][1],
{'date': date, 'status': 'INFO', 'message': 'success'})
def test_promise_resultfromlog_error(self):
promise_content = 'self.logger.error("Promise is running...\\nmessage in new line")'
self.initialisePromise(promise_content)
promise_module = self.launcher._loadPromiseModule(self.promise_name)
reload(promise_module)
promise = promise_module.RunPromise(self.promise_config)
date = datetime.now()
promise.sense()
# get all messages during the latest minute
latest_message_list = promise.getLastPromiseResultList(result_count=1)
date = datetime.strptime(date.strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')
self.assertEquals(len(latest_message_list), 1)
self.assertEquals(
latest_message_list[0][0],
{'date': date, 'status': 'ERROR',
'message': 'Promise is running...\nmessage in new line'})
self.assertEquals(
latest_message_list[0][1],
{'date': date, 'status': 'INFO', 'message': 'success'})
def test_promise_resultfromlog_no_logfolder(self):
self.log_dir = None
promise_content = "self.logger.info('Promise is running...')"
self.initialisePromise(promise_content)
promise_module = self.launcher._loadPromiseModule(self.promise_name)
reload(promise_module)
promise = promise_module.RunPromise(self.promise_config)
date = datetime.now()
promise.sense()
self.assertEquals(promise.getLogFolder(), None)
self.assertEquals(promise.getLogFile(), None)
# get all messages during the latest minute
latest_message_list = promise.getLastPromiseResultList(result_count=1)
date = datetime.strptime(date.strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')
self.assertEquals(len(latest_message_list), 1)
self.assertEquals(
latest_message_list[0][0],
{'date': date, 'status': 'INFO', 'message': 'Promise is running...'})
self.assertEquals(
latest_message_list[0][1],
{'date': date, 'status': 'INFO', 'message': 'success'})
def test_promise_resultfromlog_latest_minutes(self):
self.initialisePromise(timeout=60)
promise_module = self.launcher._loadPromiseModule(self.promise_name)
reload(promise_module)
promise = promise_module.RunPromise(self.promise_config)
# write some random logs
start_date = datetime.now()
with open(promise.getLogFile(), 'w') as f:
for i in range(0, 50):
transaction_id = '%s-%s' % (int(time.time()), random.randint(100, 999))
date = start_date - timedelta(minutes=(49 - i))
date_string = date.strftime('%Y-%m-%d %H:%M:%S')
line = "%s - INFO - %s - Promise result %s\n" % (date_string, transaction_id, i)
f.write(line)
latest_message_list = promise.getLastPromiseResultList(
latest_minute=10,
result_count=100
)
start_date = datetime.strptime(start_date.strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')
end_date_string = (start_date - timedelta(minutes=9)).strftime('%Y-%m-%d %H:%M:%S')
end_date = datetime.strptime(end_date_string, '%Y-%m-%d %H:%M:%S')
self.assertEquals(len(latest_message_list), 10)
for message in latest_message_list:
self.assertEquals(len(message), 1)
self.assertEquals(
latest_message_list[0][0],
{'date': start_date, 'status': 'INFO', 'message': 'Promise result 49'})
self.assertEquals(
latest_message_list[-1][0],
{'date': end_date, 'status': 'INFO', 'message': 'Promise result 40'})
def test_promise_resultfromlog_latest_minutes_multilog(self):
self.initialisePromise(timeout=60)
promise_module = self.launcher._loadPromiseModule(self.promise_name)
reload(promise_module)
promise = promise_module.RunPromise(self.promise_config)
# write some random logs
start_date = datetime.now()
date = start_date
line_list = []
j = 0
for i in range(0, 25):
transaction_id = '%s-%s' % (int(time.time()), random.randint(100, 999))
date_string = date.strftime('%Y-%m-%d %H:%M:%S')
line_list.append("%s - INFO - %s - Promise result %s\n" % (date_string, transaction_id, j))
date = date - timedelta(seconds=30)
j += 1
date_string = date.strftime('%Y-%m-%d %H:%M:%S')
line_list.append("%s - INFO - %s - Promise result %s\n" % (date_string, transaction_id, j))
date = date - timedelta(seconds=30)
j += 1
line_list.reverse()
with open(promise.getLogFile(), 'w') as f:
f.write('\n'.join(line_list))
latest_message_list = promise.getLastPromiseResultList(
latest_minute=10,
result_count=100
)
start_date = datetime.strptime(start_date.strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')
end_date_string = (start_date - timedelta(seconds=30*19)).strftime('%Y-%m-%d %H:%M:%S')
end_date = datetime.strptime(end_date_string, '%Y-%m-%d %H:%M:%S')
# there is 2 result line per minutes
self.assertEquals(len(latest_message_list), 10)
for message in latest_message_list:
self.assertEquals(len(message), 2)
self.assertEquals(
latest_message_list[0][1],
{'date': start_date, 'status': 'INFO', 'message': 'Promise result 0'})
self.assertEquals(
latest_message_list[-1][0],
{'date': end_date, 'status': 'INFO', 'message': 'Promise result 19'})
def test_promise_resultfromlog_result_count(self):
self.initialisePromise()
promise_module = self.launcher._loadPromiseModule(self.promise_name)
reload(promise_module)
promise = promise_module.RunPromise(self.promise_config)
# write some random logs
start_date = datetime.now()
date = start_date
line_list = []
j = 0
for i in range(0, 25):
transaction_id = '%s-%s' % (int(time.time()), random.randint(100, 999))
date_string = date.strftime('%Y-%m-%d %H:%M:%S')
line_list.append("%s - INFO - %s - Promise result %s\n" % (date_string, transaction_id, j))
date = date - timedelta(seconds=30)
j += 1
date_string = date.strftime('%Y-%m-%d %H:%M:%S')
line_list.append("%s - INFO - %s - Promise result %s\n" % (date_string, transaction_id, j))
date = date - timedelta(seconds=30)
j += 1
line_list.reverse()
with open(promise.getLogFile(), 'w') as f:
f.write('\n'.join(line_list))
# result_count = 2 will return 2 log
# max execution time is 1 min and log is writen every 30 seconds
latest_message_list = promise.getLastPromiseResultList(result_count=1)
start_date = datetime.strptime(start_date.strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')
end_date_string = (start_date - timedelta(seconds=30)).strftime('%Y-%m-%d %H:%M:%S')
end_date = datetime.strptime(end_date_string, '%Y-%m-%d %H:%M:%S')
# there is 2 result line per minutes
self.assertEquals(len(latest_message_list), 1)
self.assertEquals(
latest_message_list[0][0],
{'date': end_date, 'status': 'INFO', 'message': 'Promise result 1'})
self.assertEquals(
latest_message_list[0][1],
{'date': start_date, 'status': 'INFO', 'message': 'Promise result 0'})
def test_promise_resultfromlog_result_count_many(self):
self.initialisePromise()
promise_module = self.launcher._loadPromiseModule(self.promise_name)
reload(promise_module)
promise = promise_module.RunPromise(self.promise_config)
# write some random logs
start_date = datetime.now()
date = start_date
line_list = []
j = 0
for i in range(0, 25):
transaction_id = '%s-%s' % (int(time.time()), random.randint(100, 999))
date_string = date.strftime('%Y-%m-%d %H:%M:%S')
line_list.append("%s - INFO - %s - Promise result %s\n" % (date_string, transaction_id, j))
date = date - timedelta(seconds=30)
j += 1
date_string = date.strftime('%Y-%m-%d %H:%M:%S')
line_list.append("%s - INFO - %s - Promise result %s\n" % (date_string, transaction_id, j))
date = date - timedelta(seconds=30)
j += 1
line_list.reverse()
with open(promise.getLogFile(), 'w') as f:
f.write('\n'.join(line_list))
# result_count = 2 will return 4 log
# max execution time is 1 min and log is writen every 30 seconds
latest_message_list = promise.getLastPromiseResultList(result_count=2)
start_date = datetime.strptime(start_date.strftime('%Y-%m-%d %H:%M:%S'), '%Y-%m-%d %H:%M:%S')
end_date_string = (start_date - timedelta(seconds=30*3)).strftime('%Y-%m-%d %H:%M:%S')
end_date = datetime.strptime(end_date_string, '%Y-%m-%d %H:%M:%S')
# there is 2 result line per minutes
self.assertEquals(len(latest_message_list), 2)
for message in latest_message_list:
self.assertEquals(len(message), 2)
self.assertEquals(
latest_message_list[0][1],
{'date': start_date, 'status': 'INFO', 'message': 'Promise result 0'})
self.assertEquals(
latest_message_list[-1][0],
{'date': end_date, 'status': 'INFO', 'message': 'Promise result 3'})
latest_message_list = promise.getLastPromiseResultList(result_count=100)
# all results
self.assertEquals(len(latest_message_list), 25)
def test_promise_defaulttest(self):
promise_content = 'self.logger.info("Promise is running...\\nmessage in new line")'
self.initialisePromise(promise_content)
promise_module = self.launcher._loadPromiseModule(self.promise_name)
reload(promise_module)
promise = promise_module.RunPromise(self.promise_config)
promise.sense()
result = promise._test(result_count=1, failure_amount=1)
self.assertTrue(isinstance(result, TestResult))
self.assertEquals(result.message, 'Promise is running...\nmessage in new line\nsuccess')
self.assertEquals(result.hasFailed(), False)
def test_promise_defaulttest_failure(self):
self.initialisePromise(success=False)
promise_module = self.launcher._loadPromiseModule(self.promise_name)
reload(promise_module)
promise = promise_module.RunPromise(self.promise_config)
promise.sense()
result = promise._test(result_count=1, failure_amount=1)
self.assertTrue(isinstance(result, TestResult))
self.assertEquals(result.message, 'failed')
self.assertEquals(result.hasFailed(), True)
def test_promise_defaulttest_error_if_two_fail(self):
self.initialisePromise(success=False, timeout=1)
promise_module = self.launcher._loadPromiseModule(self.promise_name)
reload(promise_module)
promise = promise_module.RunPromise(self.promise_config)
promise.sense()
# fail if 2 errors found
result = promise._test(result_count=2, failure_amount=2)
self.assertTrue(isinstance(result, TestResult))
self.assertEquals(result.message, 'failed')
self.assertEquals(result.hasFailed(), False)
self.initialisePromise(success=False, timeout=1)
promise = promise_module.RunPromise(self.promise_config)
promise.sense()
result = promise._test(result_count=2, failure_amount=2)
self.assertEquals(result.message, 'failed')
self.assertEquals(result.hasFailed(), True)
# will continue to fail
self.initialisePromise(success=False, timeout=1)
promise = promise_module.RunPromise(self.promise_config)
promise.sense()
result = promise._test(result_count=2, failure_amount=2)
self.assertEquals(result.message, 'failed')
self.assertEquals(result.hasFailed(), True)
def test_promise_defaulttest_anomaly(self):
promise_content = 'self.logger.info("Promise is running...\\nmessage in new line")'
self.initialisePromise(promise_content)
promise_module = self.launcher._loadPromiseModule(self.promise_name)
reload(promise_module)
promise = promise_module.RunPromise(self.promise_config)
promise.sense()
result = promise._anomaly(result_count=1, failure_amount=1)
self.assertTrue(isinstance(result, AnomalyResult))
self.assertEquals(result.message, 'Promise is running...\nmessage in new line\nsuccess')
self.assertEquals(result.hasFailed(), False)
if __name__ == '__main__':
unittest.main()
slapos/tests/slapgrid.py
View file @ f7eadb26
......@@ -912,10 +912,12 @@ class TestSlapgridCPWithMasterWatchdog(MasterMixin, unittest.TestCase):
partition.software.setBuildout(BUILDOUT_RUN_CONTENT)
self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
time.sleep(1)
self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(partition.partition_path),
['.slapgrid', '.0_daemon.log', 'buildout.cfg',
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay'])
'etc', 'software_release', 'worked', '.slapos-retention-lock-delay',
'launched', 'crashed'])
daemon_log = os.path.join(partition.partition_path, '.0_daemon.log')
self.assertLogContent(daemon_log, 'Failing')
self.assertIsNotCreated(self.watchdog_banged)
......@@ -1867,14 +1869,16 @@ class TestSlapgridCPWithMasterPromise(MasterMixin, unittest.TestCase):
f.write(textwrap.dedent("""\
#!/usr/bin/env sh
touch "%s"
echo Error 1>&2
echo 'Error Promise 254554802' 1>&2
exit 127""" % worked_file))
os.chmod(succeed, 0o777)
self.assertEqual(self.grid.processComputerPartitionList(),
slapos.grid.slapgrid.SLAPGRID_PROMISE_FAIL)
self.assertTrue(os.path.isfile(worked_file))
self.assertEqual(instance.error_log[-5:], 'Error')
log_file = '%s/.slapgrid/log/instance.log' % instance.partition_path
with open(log_file) as f:
self.assertTrue('Error Promise 254554802' in f.read())
self.assertTrue(instance.error)
self.assertIsNone(instance.state)
......@@ -1986,6 +1990,58 @@ class TestSlapgridCPWithMasterPromise(MasterMixin, unittest.TestCase):
self.assertEquals(instance.error, 1)
self.assertNotEqual(instance.state, 'started')
def test_promise_run_if_partition_started_fail(self):
computer = ComputerForTest(self.software_root, self.instance_root)
with httmock.HTTMock(computer.request_handler):
instance = computer.instance_list[0]
instance.requested_state = 'started'
instance.software.setBuildout("""#!/bin/sh
exit 1
""")
self.assertEqual(self.grid.processComputerPartitionList(),
slapos.grid.slapgrid.SLAPGRID_FAIL)
self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path),
['.slapgrid', 'buildout.cfg', 'software_release',
'.slapgrid-0-error.log'])
promise_file = os.path.join(instance.partition_path, 'promise_ran')
promise = textwrap.dedent("""\
#!/usr/bin/env sh
touch "%s"
exit 127""" % promise_file)
instance.setPromise('promise_script', promise)
self.assertEqual(self.grid.processComputerPartitionList(),
slapos.grid.slapgrid.SLAPGRID_FAIL)
self.assertTrue(os.path.isfile(promise_file))
self.assertTrue(instance.error)
def test_promise_notrun_if_partition_stopped_fail(self):
computer = ComputerForTest(self.software_root, self.instance_root)
with httmock.HTTMock(computer.request_handler):
instance = computer.instance_list[0]
instance.requested_state = 'stopped'
instance.software.setBuildout("""#!/bin/sh
exit 1
""")
self.assertEqual(self.grid.processComputerPartitionList(),
slapos.grid.slapgrid.SLAPGRID_FAIL)
self.assertInstanceDirectoryListEqual(['0'])
self.assertItemsEqual(os.listdir(instance.partition_path),
['.slapgrid', 'buildout.cfg', 'software_release',
'.slapgrid-0-error.log'])
promise_file = os.path.join(instance.partition_path, 'promise_ran')
promise = textwrap.dedent("""\
#!/usr/bin/env sh
touch "%s"
exit 127""" % promise_file)
instance.setPromise('promise_script', promise)
self.assertEqual(self.grid.processComputerPartitionList(),
slapos.grid.slapgrid.SLAPGRID_FAIL)
self.assertFalse(os.path.exists(promise_file))
self.assertTrue(instance.error)
class TestSlapgridDestructionLock(MasterMixin, unittest.TestCase):
def test_retention_lock(self):
"""
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7