slapgrid.py 49.6 KB
Newer Older
1
# -*- coding: utf-8 -*-
2
# vim: set et sts=2:
Łukasz Nowak's avatar
Łukasz Nowak committed
3 4
##############################################################################
#
5 6
# Copyright (c) 2010, 2011, 2012 Vifib SARL and Contributors.
# All Rights Reserved.
Łukasz Nowak's avatar
Łukasz Nowak committed
7 8 9 10 11
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
12
# guarantees and support are strongly advised to contract a Free Software
Łukasz Nowak's avatar
Łukasz Nowak committed
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################
30

Cédric de Saint Martin's avatar
Cédric de Saint Martin committed
31 32
import argparse
import ConfigParser
Łukasz Nowak's avatar
Łukasz Nowak committed
33 34 35
import logging
import os
import pkg_resources
36
import random
Cédric de Saint Martin's avatar
Cédric de Saint Martin committed
37 38
import socket
import StringIO
39
import subprocess
Cédric de Saint Martin's avatar
Cédric de Saint Martin committed
40 41 42 43
import sys
import tempfile
import time
import traceback
Łukasz Nowak's avatar
Łukasz Nowak committed
44
import warnings
45

Łukasz Nowak's avatar
Łukasz Nowak committed
46
if sys.version_info < (2, 6):
Marco Mariani's avatar
Marco Mariani committed
47
  warnings.warn('Used python version (%s) is old and has problems with'
Łukasz Nowak's avatar
Łukasz Nowak committed
48 49
      ' IPv6 connections' % sys.version.split('\n')[0])

Marco Mariani's avatar
Marco Mariani committed
50 51
from lxml import etree

Cédric de Saint Martin's avatar
Cédric de Saint Martin committed
52 53
from slapos.slap.slap import NotFoundError
from slapos.slap.slap import ServerError
Marco Mariani's avatar
Marco Mariani committed
54
from slapos.grid.exception import BuildoutFailedError
55
from slapos.grid.SlapObject import Software, Partition
Marco Mariani's avatar
Marco Mariani committed
56
from slapos.grid.svcbackend import launchSupervisord
57
from slapos.grid.utils import (md5digest, createPrivateDirectory, dropPrivileges,
Marco Mariani's avatar
Marco Mariani committed
58 59
                               setRunning, setFinished, SlapPopen, updateFile)
import slapos.slap
Łukasz Nowak's avatar
Łukasz Nowak committed
60 61


Cédric de Saint Martin's avatar
Cédric de Saint Martin committed
62
# XXX: should be moved to SLAP library
63
COMPUTER_PARTITION_DESTROYED_STATE = 'destroyed'
64 65
COMPUTER_PARTITION_STARTED_STATE = 'started'
COMPUTER_PARTITION_STOPPED_STATE = 'stopped'
Łukasz Nowak's avatar
Łukasz Nowak committed
66

67 68 69 70 71
# Global variables about return state of slapgrid
SLAPGRID_SUCCESS = 0
SLAPGRID_FAIL = 1
SLAPGRID_PROMISE_FAIL = 2

72
# XXX hardcoded watchdog_path
73
WATCHDOG_PATH = '/opt/slapos/bin/slapos-watchdog'
74

75 76
COMPUTER_PARTITION_TIMESTAMP_FILENAME = '.timestamp'

77 78 79 80 81

class _formatXMLError(Exception):
  pass


82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
def check_missing_parameters(options):
  required = set([
      'computer_id',
      'instance_root',
      'master_url',
      'software_root',
  ])

  if 'key_file' in options:
    required.add('certificate_repository_path')
    required.add('cert_file')
  if 'cert_file' in options:
    required.add('certificate_repository_path')
    required.add('key_file')

  missing = required.difference(options)

  if missing:
    raise RuntimeError('Missing mandatory parameters: %s' % ', '.join(sorted(missing)))


103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
def check_missing_files(options):
  req_files = [
          options.get('key_file'),
          options.get('cert_file'),
          options.get('master_ca_file'),
          options.get('shacache-cert-file'),
          options.get('shacache-key-file'),
          options.get('shadir-cert-file'),
          options.get('shadir-key-file'),
          options.get('signature_private_key_file')
          ]

  req_dirs = [
          options.get('certificate_repository_path')
          ]

  for f in req_files:
    if f and not os.path.exists(f):
        raise RuntimeError('File %r does not exist.' % f)

  for d in req_dirs:
    if d and not os.path.isdir(d):
      raise RuntimeError('Directory %r does not exist' % d)


128
def parse_arguments_merge_config(*argument_tuple):
129
  """Parse arguments and return options dictionary merged with the config file."""
130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174

  ap = argparse.ArgumentParser()
  ap.add_argument('--instance-root',
                  help='The instance root directory location.')
  ap.add_argument('--software-root',
                  help='The software_root directory location.')
  ap.add_argument('--master-url',
                  help='The master server URL. Mandatory.')
  ap.add_argument('--computer-id',
                  help='The computer id defined in the server.')
  ap.add_argument('--supervisord-socket',
                  help='The socket supervisor will use.')
  ap.add_argument('--supervisord-configuration-path',
                  help='The location where supervisord configuration will be stored.')
  ap.add_argument('--buildout', default=None,
                  help='Location of buildout binary.')
  ap.add_argument('--pidfile',
                  help='The location where pidfile will be created.')
  ap.add_argument('--logfile',
                  help='The location where slapgrid logfile will be created.')
  ap.add_argument('--key_file',
                  help='SSL Authorisation key file.')
  ap.add_argument('--cert_file',
                  help='SSL Authorisation certificate file.')
  ap.add_argument('--signature_private_key_file',
                  help='Signature private key file.')
  ap.add_argument('--master_ca_file',
                  help='Root certificate of SlapOS master key.')
  ap.add_argument('--certificate_repository_path',
                  help='Path to directory where downloaded certificates would be stored.')
  ap.add_argument('-v', '--verbose', action='store_true',
                  help='Be verbose.')
  ap.add_argument('--maximum-periodicity', type=int, default=None,
                  help='Periodicity at which buildout should be run in instance.')
  ap.add_argument('--promise-timeout', type=int, default=3,
                  help='Promise timeout in seconds.')
  ap.add_argument('--now', action='store_true',
                  help='Launch slapgrid without delay. Default behavior.')
  ap.add_argument('--all', action='store_true',
                  help='Launch slapgrid to process all Softare Releases '
                       'and/or Computer Partitions.')
  ap.add_argument('--only-sr',
                  help='Force the update of a single software release (use url hash), '
                       'even if is already installed. This option will make all others '
                       'sofware releases be ignored.')
175
  ap.add_argument('--only-cp',
176 177 178 179 180 181
                  help='Update a single or a list of computer partitions '
                       '(ie.:slappartX, slappartY), '
                       'this option will make all others computer partitions be ignored.')

  ap.add_argument('configuration_file', type=argparse.FileType(),
                  help='SlapOS configuration file.')
182 183

  # Deprecated options
184 185 186 187 188 189 190 191 192 193
  ap.add_argument('-c', '--console', action='store_true',
                  help="Deprecated, doesn't do anything.")
  ap.add_argument('--develop', action='store_true',
                  help='Deprecated, same as --all.')
  ap.add_argument('--only_sr',
                  help='Deprecated, same as --only-sr.')
  ap.add_argument('--only_cp',
                  help='Deprecated, same as --only-cp.')
  ap.add_argument('--maximal_delay',
                  help='Deprecated. Will only work from configuration file in the future.')
194

Marco Mariani's avatar
Marco Mariani committed
195
  if not argument_tuple:
196
    args = ap.parse_args()
Łukasz Nowak's avatar
Łukasz Nowak committed
197
  else:
198
    args = ap.parse_args(list(argument_tuple))
199

Marco Mariani's avatar
Marco Mariani committed
200
  options = {}
201 202
  config = ConfigParser.SafeConfigParser()
  config.readfp(args.configuration_file)
203

204 205 206 207 208 209 210 211 212 213 214
  options = dict(config.items('slapos'))
  if config.has_section('networkcache'):
    options.update(dict(config.items('networkcache')))
  for key, value in vars(args).iteritems():
    if value is not None:
      options[key] = value

  return options


def setup_logger(options):
Marco Mariani's avatar
Marco Mariani committed
215
  if options['verbose']:
Łukasz Nowak's avatar
Łukasz Nowak committed
216 217 218
    level = logging.DEBUG
  else:
    level = logging.INFO
219 220 221
  logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s',
                      level=level,
                      datefmt='%Y-%m-%dT%H:%M:%S')
Marco Mariani's avatar
Marco Mariani committed
222 223
  if options.get('logfile'):
    console = logging.FileHandler(options['logfile'])
224 225 226 227 228
    console.setLevel(level)
    console.setFormatter(logging.Formatter(
        '%(asctime)s %(name)-18s: %(levelname)-8s %(message)s'))
    logging.getLogger('').addHandler(console)

229 230
  logger = logging.getLogger(__name__)
  return logger
231 232


233
def random_delay(options, logger):
234 235 236 237
  """
  Sleep for a random time to avoid SlapOS Master being DDOSed by an army of
  SlapOS Nodes configured with cron.
  """
238 239
  if options['now']:
    # XXX-Cedric: deprecate '--now'
240 241
    return

242
  maximal_delay = int(options.get('maximal_delay', '0'))
243 244
  if maximal_delay:
    duration = random.randint(1, maximal_delay)
245 246
    logger.info('Sleeping for %s seconds. To disable this feature, ' \
                'check --now parameter in slapgrid help.' % duration)
247 248 249 250
    time.sleep(duration)



251 252
def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
  """Returns a new instance of slapgrid.Slapgrid created with argument+config parameters.
253
     Also returns the pidfile path, and configures logger.
254 255 256
  """
  options = parse_arguments_merge_config(*argument_tuple)

257
  logger = setup_logger(options)
258

259
  check_missing_parameters(options)
260
  check_missing_files(options)
Łukasz Nowak's avatar
Łukasz Nowak committed
261

Marco Mariani's avatar
Marco Mariani committed
262
  if options.get('all'):
Marco Mariani's avatar
Marco Mariani committed
263
    options['develop'] = True
264

Marco Mariani's avatar
Marco Mariani committed
265 266
  if options.get('maximum_periodicity') is not None:
    options['force_periodicity'] = True
267

Łukasz Nowak's avatar
Łukasz Nowak committed
268
  # Supervisord configuration location
Marco Mariani's avatar
Marco Mariani committed
269 270 271
  if not options.get('supervisord_configuration_path'):
    options['supervisord_configuration_path'] = \
      os.path.join(options['instance_root'], 'etc', 'supervisord.conf')
Łukasz Nowak's avatar
Łukasz Nowak committed
272
  # Supervisord socket
Marco Mariani's avatar
Marco Mariani committed
273 274 275
  if not options.get('supervisord_socket'):
    options['supervisord_socket'] = \
      os.path.join(options['instance_root'], 'supervisord.socket')
Yingjie Xu's avatar
Yingjie Xu committed
276

277 278
  # Parse cache / binary cache options
  # Backward compatibility about "binary-cache-url-blacklist" deprecated option
Marco Mariani's avatar
Marco Mariani committed
279 280 281 282 283 284
  if options.get("binary-cache-url-blacklist") and not \
      options.get("download-from-binary-cache-url-blacklist"):
    options["download-from-binary-cache-url-blacklist"] = \
        options["binary-cache-url-blacklist"]
  options["download-from-binary-cache-url-blacklist"] = [
      url.strip() for url in options.get(
285
          "download-from-binary-cache-url-blacklist", "").split('\n') if url]
Marco Mariani's avatar
Marco Mariani committed
286 287
  options["upload-to-binary-cache-url-blacklist"] = [
      url.strip() for url in options.get(
288
          "upload-to-binary-cache-url-blacklist", "").split('\n') if url]
289

290
  random_delay(options, logger=logger)
291

292
  slapgrid_object = create_slapgrid_object(options, logger=logger)
293 294 295 296 297

  return slapgrid_object, options.get('pidfile')



298
def create_slapgrid_object(options, logger):
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
  signature_certificate_list = None
  if 'signature-certificate-list' in options:
    cert_marker = '-----BEGIN CERTIFICATE-----'
    signature_certificate_list = [
            cert_marker + '\n' + q.strip()
            for q in options['signature-certificate-list'].split(cert_marker)
            if q.strip()
            ]

  op = options
  return Slapgrid(software_root=op['software_root'],
                  instance_root=op['instance_root'],
                  master_url=op['master_url'],
                  computer_id=op['computer_id'],
                  supervisord_socket=op['supervisord_socket'],
                  supervisord_configuration_path=op['supervisord_configuration_path'],
315 316 317 318
                  buildout=op.get('buildout'),
                  logger=logger,
                  force_periodicity = op.get('force_periodicity', False),
                  maximum_periodicity = op.get('maximum_periodicity', 86400),
319 320 321 322 323 324 325 326 327 328 329 330 331 332
                  key_file=op.get('key_file'),
                  cert_file=op.get('cert_file'),
                  signature_private_key_file=op.get('signature_private_key_file'),
                  signature_certificate_list=signature_certificate_list,
                  download_binary_cache_url=op.get('download-binary-cache-url'),
                  upload_binary_cache_url=op.get('upload-binary-cache-url'),
                  download_from_binary_cache_url_blacklist=\
                      op.get('download-from-binary-cache-url-blacklist', []),
                  upload_to_binary_cache_url_blacklist=\
                      op.get('upload-to-binary-cache-url-blacklist', []),
                  upload_cache_url=op.get('upload-cache-url'),
                  download_binary_dir_url=op.get('download-binary-dir-url'),
                  upload_binary_dir_url=op.get('upload-binary-dir-url'),
                  upload_dir_url=op.get('upload-dir-url'),
333 334
                  master_ca_file=op.get('master_ca_file'),
                  certificate_repository_path=op.get('certificate_repository_path'),
335 336 337 338 339 340 341 342 343
                  promise_timeout=op['promise_timeout'],
                  shacache_cert_file=op.get('shacache-cert-file'),
                  shacache_key_file=op.get('shacache-key-file'),
                  shadir_cert_file=op.get('shadir-cert-file'),
                  shadir_key_file=op.get('shadir-key-file'),
                  develop=op.get('develop', False),
                  # Try to fetch from deprecated argument
                  software_release_filter_list=op.get('only-sr', op.get('only_sr')),
                  # Try to fetch from deprecated argument
344
                  computer_partition_filter_list=op.get('only-cp', op.get('only_cp')))
Łukasz Nowak's avatar
Łukasz Nowak committed
345 346


347
def realRun(argument_tuple, method):
348
  slapgrid_object, pidfile = parseArgumentTupleAndReturnSlapgridObject(*argument_tuple)
Łukasz Nowak's avatar
Łukasz Nowak committed
349
  if pidfile:
350
    setRunning(logger=slapgrid_object.logger, pid_file=pidfile)
Łukasz Nowak's avatar
Łukasz Nowak committed
351
  try:
352 353
    failed = False
    failed_promise = False
354 355 356 357 358 359
    # Quite complicated way to figure out if everything went fine
    return_value = getattr(slapgrid_object, method)()
    if return_value == SLAPGRID_FAIL:
      failed = True
    if return_value == SLAPGRID_PROMISE_FAIL:
      failed_promise = True
Łukasz Nowak's avatar
Łukasz Nowak committed
360 361 362
  finally:
    if pidfile:
      setFinished(pidfile)
363 364 365 366 367
  if failed:
    sys.exit(SLAPGRID_FAIL)
  if failed_promise:
    sys.exit(SLAPGRID_PROMISE_FAIL)
  sys.exit(SLAPGRID_SUCCESS)
Łukasz Nowak's avatar
Łukasz Nowak committed
368 369 370


def runSoftwareRelease(*argument_tuple):
371
  """Hook for entry point to process Software Releases"""
372
  realRun(argument_tuple, 'processSoftwareReleaseList')
Łukasz Nowak's avatar
Łukasz Nowak committed
373 374 375


def runComputerPartition(*argument_tuple):
376
  """Hook for entry point to process Computer Partitions"""
377
  realRun(argument_tuple, 'processComputerPartitionList')
Łukasz Nowak's avatar
Łukasz Nowak committed
378 379 380


def runUsageReport(*argument_tuple):
381
  """Hook for entry point to process Usage Reports"""
382
  realRun(argument_tuple, 'agregateAndSendUsage')
Łukasz Nowak's avatar
Łukasz Nowak committed
383 384 385 386 387 388


class Slapgrid(object):
  """ Main class for SlapGrid. Fetches and processes informations from master
  server and pushes usage information to master server.
  """
Antoine Catton's avatar
Antoine Catton committed
389 390 391 392

  class PromiseError(Exception):
    pass

Łukasz Nowak's avatar
Łukasz Nowak committed
393 394 395 396 397 398 399
  def __init__(self,
               software_root,
               instance_root,
               master_url,
               computer_id,
               supervisord_socket,
               supervisord_configuration_path,
400
               buildout,
401
               logger,
402 403
               force_periodicity=False,
               maximum_periodicity=86400,
Łukasz Nowak's avatar
Łukasz Nowak committed
404 405
               key_file=None,
               cert_file=None,
406
               signature_private_key_file=None,
Yingjie Xu's avatar
Yingjie Xu committed
407 408 409
               signature_certificate_list=None,
               download_binary_cache_url=None,
               upload_binary_cache_url=None,
410 411
               download_from_binary_cache_url_blacklist=None,
               upload_to_binary_cache_url_blacklist=None,
412
               upload_cache_url=None,
Yingjie Xu's avatar
Yingjie Xu committed
413 414
               download_binary_dir_url=None,
               upload_binary_dir_url=None,
415
               upload_dir_url=None,
Łukasz Nowak's avatar
Łukasz Nowak committed
416 417
               master_ca_file=None,
               certificate_repository_path=None,
418 419 420 421
               promise_timeout=3,
               shacache_cert_file=None,
               shacache_key_file=None,
               shadir_cert_file=None,
422
               shadir_key_file=None,
423
               develop=False,
424
               software_release_filter_list=None,
425 426
               computer_partition_filter_list=None,
               ):
Łukasz Nowak's avatar
Łukasz Nowak committed
427 428 429 430 431 432 433 434 435 436 437 438
    """Makes easy initialisation of class parameters"""
    # Parses arguments
    self.software_root = os.path.abspath(software_root)
    self.instance_root = os.path.abspath(instance_root)
    self.master_url = master_url
    self.computer_id = computer_id
    self.supervisord_socket = supervisord_socket
    self.supervisord_configuration_path = supervisord_configuration_path
    self.key_file = key_file
    self.cert_file = cert_file
    self.master_ca_file = master_ca_file
    self.certificate_repository_path = certificate_repository_path
439
    self.signature_private_key_file = signature_private_key_file
Yingjie Xu's avatar
Yingjie Xu committed
440 441 442
    self.signature_certificate_list = signature_certificate_list
    self.download_binary_cache_url = download_binary_cache_url
    self.upload_binary_cache_url = upload_binary_cache_url
443 444 445 446
    self.download_from_binary_cache_url_blacklist = \
        download_from_binary_cache_url_blacklist
    self.upload_to_binary_cache_url_blacklist = \
        upload_to_binary_cache_url_blacklist
447
    self.upload_cache_url = upload_cache_url
Yingjie Xu's avatar
Yingjie Xu committed
448 449
    self.download_binary_dir_url = download_binary_dir_url
    self.upload_binary_dir_url = upload_binary_dir_url
450
    self.upload_dir_url = upload_dir_url
451 452 453 454
    self.shacache_cert_file = shacache_cert_file
    self.shacache_key_file = shacache_key_file
    self.shadir_cert_file = shadir_cert_file
    self.shadir_key_file = shadir_key_file
455
    self.logger = logger
Łukasz Nowak's avatar
Łukasz Nowak committed
456
    # Creates objects from slap module
Marco Mariani's avatar
Marco Mariani committed
457
    self.slap = slapos.slap.slap()
Łukasz Nowak's avatar
Łukasz Nowak committed
458 459 460 461 462
    self.slap.initializeConnection(self.master_url, key_file=self.key_file,
        cert_file=self.cert_file, master_ca_file=self.master_ca_file)
    self.computer = self.slap.registerComputer(self.computer_id)
    # Defines all needed paths
    self.supervisord_configuration_directory = \
Marco Mariani's avatar
Marco Mariani committed
463
        os.path.join(self.instance_root, 'etc', 'supervisord.conf.d')
464
    self.buildout = buildout
465
    self.promise_timeout = promise_timeout
466
    self.develop = develop
467
    if software_release_filter_list is not None:
Cédric de Saint Martin's avatar
Cédric de Saint Martin committed
468 469
      self.software_release_filter_list = \
          software_release_filter_list.split(",")
470
    else:
471
      self.software_release_filter_list = []
472 473
    self.computer_partition_filter_list = []
    if computer_partition_filter_list is not None:
Cédric de Saint Martin's avatar
Cédric de Saint Martin committed
474 475
      self.computer_partition_filter_list = \
          computer_partition_filter_list.split(",")
476 477
    self.maximum_periodicity = maximum_periodicity
    self.force_periodicity = force_periodicity
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
478 479

  def getWatchdogLine(self):
480
    invocation_list = [WATCHDOG_PATH]
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
481
    invocation_list.append("--master-url '%s' " % self.master_url)
482
    if self.certificate_repository_path:
483 484
      invocation_list.append("--certificate-repository-path '%s'" \
                               % self.certificate_repository_path)
Cédric Le Ninivin's avatar
Cédric Le Ninivin committed
485 486
    invocation_list.append("--computer-id '%s'" % self.computer_id)
    return ' '.join(invocation_list)
Łukasz Nowak's avatar
Łukasz Nowak committed
487 488 489 490 491 492 493

  def checkEnvironmentAndCreateStructure(self):
    """Checks for software_root and instance_root existence, then creates
       needed files and directories.
    """
    # Checks for software_root and instance_root existence
    if not os.path.isdir(self.software_root):
494
      raise OSError('%s does not exist.' % self.software_root)
Łukasz Nowak's avatar
Łukasz Nowak committed
495
    if not os.path.isdir(self.instance_root):
496
      raise OSError('%s does not exist.' % self.instance_root)
Łukasz Nowak's avatar
Łukasz Nowak committed
497
    # Creates everything needed
Marco Mariani's avatar
Marco Mariani committed
498

499 500 501 502
    # Creates instance_root structure
    createPrivateDirectory(os.path.join(self.instance_root, 'var'))
    createPrivateDirectory(os.path.join(self.instance_root, 'var', 'log'))
    createPrivateDirectory(os.path.join(self.instance_root, 'var', 'run'))
Marco Mariani's avatar
Marco Mariani committed
503 504

    createPrivateDirectory(os.path.join(self.instance_root, 'etc'))
505
    createPrivateDirectory(self.supervisord_configuration_directory)
506

507 508 509 510 511 512 513 514 515 516 517 518 519 520
    # Creates supervisord configuration
    updateFile(self.supervisord_configuration_path,
      pkg_resources.resource_stream(__name__,
        'templates/supervisord.conf.in').read() % {
            'supervisord_configuration_directory': self.supervisord_configuration_directory,
            'supervisord_socket': os.path.abspath(self.supervisord_socket),
            'supervisord_loglevel': 'info',
            'supervisord_logfile': os.path.abspath(os.path.join(self.instance_root, 'var', 'log', 'supervisord.log')),
            'supervisord_logfile_maxbytes': '50MB',
            'supervisord_nodaemon': 'false',
            'supervisord_pidfile': os.path.abspath(os.path.join(self.instance_root, 'var', 'run', 'supervisord.pid')),
            'supervisord_logfile_backups': '10',
            'watchdog_command': self.getWatchdogLine(),
            })
Łukasz Nowak's avatar
Łukasz Nowak committed
521 522 523

  def getComputerPartitionList(self):
    try:
524
      return self.computer.getComputerPartitionList()
Marco Mariani's avatar
Marco Mariani committed
525 526
    except socket.error as exc:
      self.logger.fatal(exc)
527
      raise
Łukasz Nowak's avatar
Łukasz Nowak committed
528 529 530 531 532

  def processSoftwareReleaseList(self):
    """Will process each Software Release.
    """
    self.checkEnvironmentAndCreateStructure()
533
    self.logger.info('Processing software releases...')
534
    # Boolean to know if every instance has correctly been deployed
Łukasz Nowak's avatar
Łukasz Nowak committed
535 536
    clean_run = True
    for software_release in self.computer.getSoftwareReleaseList():
Łukasz Nowak's avatar
Łukasz Nowak committed
537
      state = software_release.getState()
Łukasz Nowak's avatar
Łukasz Nowak committed
538 539
      try:
        software_release_uri = software_release.getURI()
Marco Mariani's avatar
Marco Mariani committed
540
        url_hash = md5digest(software_release_uri)
541
        software_path = os.path.join(self.software_root, url_hash)
Łukasz Nowak's avatar
Łukasz Nowak committed
542 543
        software = Software(url=software_release_uri,
            software_root=self.software_root,
544
            buildout=self.buildout,
545
            logger=self.logger,
546
            signature_private_key_file=self.signature_private_key_file,
Yingjie Xu's avatar
Yingjie Xu committed
547 548 549
            signature_certificate_list=self.signature_certificate_list,
            download_binary_cache_url=self.download_binary_cache_url,
            upload_binary_cache_url=self.upload_binary_cache_url,
550 551 552 553
            download_from_binary_cache_url_blacklist=\
                self.download_from_binary_cache_url_blacklist,
            upload_to_binary_cache_url_blacklist=\
                self.upload_to_binary_cache_url_blacklist,
554
            upload_cache_url=self.upload_cache_url,
Yingjie Xu's avatar
Yingjie Xu committed
555 556
            download_binary_dir_url=self.download_binary_dir_url,
            upload_binary_dir_url=self.upload_binary_dir_url,
557 558 559 560
            upload_dir_url=self.upload_dir_url,
            shacache_cert_file=self.shacache_cert_file,
            shacache_key_file=self.shacache_key_file,
            shadir_cert_file=self.shadir_cert_file,
Łukasz Nowak's avatar
Łukasz Nowak committed
561 562
            shadir_key_file=self.shadir_key_file)
        if state == 'available':
563
          completed_tag = os.path.join(software_path, '.completed')
564
          if self.develop or (not os.path.exists(completed_tag) and \
565
                 len(self.software_release_filter_list) == 0) or \
566
                 url_hash in self.software_release_filter_list or \
Marco Mariani's avatar
Marco Mariani committed
567
                 url_hash in (md5digest(uri) for uri in self.software_release_filter_list):
568 569 570 571 572
            try:
              software_release.building()
            except NotFoundError:
              pass
            software.install()
Marco Mariani's avatar
Marco Mariani committed
573 574
            with open(completed_tag, 'w') as fout:
              fout.write(time.asctime())
Łukasz Nowak's avatar
Łukasz Nowak committed
575
        elif state == 'destroyed':
576
          if os.path.exists(software_path):
577
            self.logger.info('Destroying %r...' % software_release_uri)
578
            software.destroy()
579
            self.logger.info('Destroyed %r.' % software_release_uri)
580
      # Send log before exiting
Łukasz Nowak's avatar
Łukasz Nowak committed
581
      except (SystemExit, KeyboardInterrupt):
582
        software_release.error(traceback.format_exc(), logger=self.logger)
Łukasz Nowak's avatar
Łukasz Nowak committed
583
        raise
584 585

      # Buildout failed: send log but don't print it to output (already done)
Marco Mariani's avatar
Marco Mariani committed
586
      except BuildoutFailedError as exc:
587 588
        clean_run = False
        try:
589
          software_release.error(exc, logger=self.logger)
590 591 592
        except (SystemExit, KeyboardInterrupt):
          raise
        except Exception:
593 594
          self.logger.error('Problem while reporting error, continuing:\n%s' %
                            traceback.format_exc())
595 596

      # For everything else: log it, send it, continue.
Łukasz Nowak's avatar
Łukasz Nowak committed
597
      except Exception:
Marco Mariani's avatar
Marco Mariani committed
598
        exc = traceback.format_exc()
599
        self.logger.error(exc)
600
        software_release.error(exc, logger=self.logger)
Łukasz Nowak's avatar
Łukasz Nowak committed
601 602
        clean_run = False
      else:
Łukasz Nowak's avatar
Łukasz Nowak committed
603
        if state == 'available':
604 605 606 607
          try:
            software_release.available()
          except NotFoundError:
            pass
Łukasz Nowak's avatar
Łukasz Nowak committed
608
        elif state == 'destroyed':
609 610
          try:
            software_release.destroyed()
611 612
          except (NotFoundError, ServerError):
            print traceback.format_exc()
613
    self.logger.info('Finished software releases.')
614 615 616 617 618

    # Return success value
    if not clean_run:
      return SLAPGRID_FAIL
    return SLAPGRID_SUCCESS
Łukasz Nowak's avatar
Łukasz Nowak committed
619

620

Łukasz Nowak's avatar
Łukasz Nowak committed
621 622
  def _launchSupervisord(self):
    launchSupervisord(self.supervisord_socket,
Marco Mariani's avatar
Marco Mariani committed
623 624
                      self.supervisord_configuration_path,
                      logger=self.logger)
Łukasz Nowak's avatar
Łukasz Nowak committed
625

Antoine Catton's avatar
Antoine Catton committed
626
  def _checkPromises(self, computer_partition):
627
    self.logger.info("Checking promises...")
Marco Mariani's avatar
Marco Mariani committed
628
    instance_path = os.path.join(self.instance_root, computer_partition.getId())
Antoine Catton's avatar
Antoine Catton committed
629 630 631 632 633 634 635 636

    uid, gid = None, None
    stat_info = os.stat(instance_path)

    #stat sys call to get statistics informations
    uid = stat_info.st_uid
    gid = stat_info.st_gid

637
    promise_present = False
Antoine Catton's avatar
Antoine Catton committed
638 639 640 641
    # Get the list of promises
    promise_dir = os.path.join(instance_path, 'etc', 'promise')
    if os.path.exists(promise_dir) and os.path.isdir(promise_dir):
      # Check whether every promise is kept
Marco Mariani's avatar
Marco Mariani committed
642
      for promise in os.listdir(promise_dir):
643
        promise_present = True
Antoine Catton's avatar
Antoine Catton committed
644

Antoine Catton's avatar
Antoine Catton committed
645 646 647
        command = [os.path.join(promise_dir, promise)]

        promise = os.path.basename(command[0])
648
        self.logger.info("Checking promise %r.", promise)
Antoine Catton's avatar
Antoine Catton committed
649

650
        process_handler = subprocess.Popen(command,
651
                                           preexec_fn=lambda: dropPrivileges(uid, gid, logger=self.logger),
Marco Mariani's avatar
Marco Mariani committed
652
                                           cwd=instance_path,
Marco Mariani's avatar
Marco Mariani committed
653
                                           env=None if sys.platform == 'cygwin' else {},
Marco Mariani's avatar
Marco Mariani committed
654 655 656
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE,
                                           stdin=subprocess.PIPE)
657 658 659
        process_handler.stdin.flush()
        process_handler.stdin.close()
        process_handler.stdin = None
Antoine Catton's avatar
Antoine Catton committed
660 661 662 663

        time.sleep(self.promise_timeout)

        if process_handler.poll() is None:
664
          process_handler.terminate()
Antoine Catton's avatar
Antoine Catton committed
665 666 667 668 669
          raise Slapgrid.PromiseError("The promise %r timed out" % promise)
        elif process_handler.poll() != 0:
          stderr = process_handler.communicate()[1]
          if stderr is None:
            stderr = 'No error output from %r.' % promise
Antoine Catton's avatar
Antoine Catton committed
670 671
          else:
            stderr = 'Promise %r:' % promise + stderr
Antoine Catton's avatar
Antoine Catton committed
672 673
          raise Slapgrid.PromiseError(stderr)

Antoine Catton's avatar
Antoine Catton committed
674 675 676
    if not promise_present:
      self.logger.info("No promise.")

677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693
  def processComputerPartition(self, computer_partition):
    """
    Process a Computer Partition, depending on its state
    """
    computer_partition_id = computer_partition.getId()

    # Sanity checks before processing
    # Those values should not be None or empty string or any falsy value
    if not computer_partition_id:
      raise ValueError('Computer Partition id is empty.')

    # Check if we defined explicit list of partitions to process.
    # If so, if current partition not in this list, skip.
    if len(self.computer_partition_filter_list) > 0 and \
         (computer_partition_id not in self.computer_partition_filter_list):
      return

694
    self.logger.info('Processing Computer Partition %s...' % computer_partition_id)
695

696 697 698
    instance_path = os.path.join(self.instance_root, computer_partition_id)

    # Try to get partition timestamp (last modification date)
699 700 701 702
    timestamp_path = os.path.join(
        instance_path,
        COMPUTER_PARTITION_TIMESTAMP_FILENAME
    )
703 704 705 706 707 708
    parameter_dict = computer_partition.getInstanceParameterDict()
    if 'timestamp' in parameter_dict:
      timestamp = parameter_dict['timestamp']
    else:
      timestamp = None

709 710 711 712 713 714 715
    try:
      software_url = computer_partition.getSoftwareRelease().getURI()
    except NotFoundError:
      # Problem with instance: SR URI not set.
      # Try to process it anyway, it may need to be deleted.
      software_url = None
    try:
Marco Mariani's avatar
Marco Mariani committed
716
      software_path = os.path.join(self.software_root, md5digest(software_url))
717 718 719 720 721
    except TypeError:
      # Problem with instance: SR URI not set.
      # Try to process it anyway, it may need to be deleted.
      software_path = None

722
    periodicity = self.maximum_periodicity
723
    if software_path:
724 725
      # Get periodicity from periodicity file if not forced
      if not self.force_periodicity:
726
        periodicity_path = os.path.join(software_path, 'periodicity')
727 728
        if os.path.exists(periodicity_path):
          try:
729
            periodicity = int(open(periodicity_path).read())
730 731
          except ValueError:
            os.remove(periodicity_path)
732
            self.logger.error(traceback.format_exc())
733 734 735 736 737 738 739 740 741 742 743

    # Check if timestamp from server is more recent than local one.
    # If not: it's not worth processing this partition (nothing has
    # changed).
    if computer_partition_id not in self.computer_partition_filter_list and \
        (not self.develop) and os.path.exists(timestamp_path):
      old_timestamp = open(timestamp_path).read()
      last_runtime = int(os.path.getmtime(timestamp_path))
      if timestamp:
        try:
          if int(timestamp) <= int(old_timestamp):
744 745
            if computer_partition.getState() != COMPUTER_PARTITION_STARTED_STATE:
              return
746 747 748 749
            # Check periodicity, i.e if periodicity is one day, partition
            # should be processed at least every day.
            # Only do it for "started" instances
            if int(time.time()) <= (last_runtime + periodicity):
750
              self.logger.info('Partition already up-to-date, skipping.')
751
              return
752 753 754 755
            else:
              # Periodicity forced processing this partition. Removing
              # the timestamp file in case it fails.
              os.remove(timestamp_path)
756 757
        except ValueError:
          os.remove(timestamp_path)
758
          self.logger.error(traceback.format_exc())
759 760 761 762 763 764 765 766 767 768 769 770 771 772

    local_partition = Partition(
      software_path=software_path,
      instance_path=instance_path,
      supervisord_partition_configuration_path=os.path.join(
        self.supervisord_configuration_directory, '%s.conf' %
        computer_partition_id),
      supervisord_socket=self.supervisord_socket,
      computer_partition=computer_partition,
      computer_id=self.computer_id,
      partition_id=computer_partition_id,
      server_url=self.master_url,
      software_release_url=software_url,
      certificate_repository_path=self.certificate_repository_path,
773 774
      buildout=self.buildout,
      logger=self.logger)
775 776

    computer_partition_state = computer_partition.getState()
777
    if computer_partition_state == COMPUTER_PARTITION_STARTED_STATE:
778 779 780 781 782
      local_partition.install()
      computer_partition.available()
      local_partition.start()
      self._checkPromises(computer_partition)
      computer_partition.started()
783
    elif computer_partition_state == COMPUTER_PARTITION_STOPPED_STATE:
784 785 786 787 788 789 790 791
      try:
        local_partition.install()
        computer_partition.available()
      except Exception:
        raise
      finally:
        # Instance has to be stopped even if buildout/reporting is wrong.
        local_partition.stop()
792
      computer_partition.stopped()
793
    elif computer_partition_state == COMPUTER_PARTITION_DESTROYED_STATE:
794 795 796 797
      local_partition.stop()
      try:
        computer_partition.stopped()
      except (SystemExit, KeyboardInterrupt):
798
        computer_partition.error(traceback.format_exc(), logger=self.logger)
799 800 801 802 803 804
        raise
      except Exception:
        pass
    else:
      error_string = "Computer Partition %r has unsupported state: %s" % \
        (computer_partition_id, computer_partition_state)
805
      computer_partition.error(error_string, logger=self.logger)
806 807 808 809
      raise NotImplementedError(error_string)

    # If partition has been successfully processed, write timestamp
    if timestamp:
810 811 812 813
      timestamp_path = os.path.join(
          instance_path,
          COMPUTER_PARTITION_TIMESTAMP_FILENAME
      )
814 815
      open(timestamp_path, 'w').write(timestamp)

816
  def FilterComputerPartitionList(self, computer_partition_list):
Cédric de Saint Martin's avatar
Cédric de Saint Martin committed
817
    """
818
    Try to filter valid partitions to be processed from free partitions.
Łukasz Nowak's avatar
Łukasz Nowak committed
819
    """
820 821
    filtered_computer_partition_list = []
    for computer_partition in computer_partition_list:
Łukasz Nowak's avatar
Łukasz Nowak committed
822
      try:
823 824 825 826 827 828 829
        computer_partition_path = os.path.join(self.instance_root,
            computer_partition.getId())
        if not os.path.exists(computer_partition_path):
          raise NotFoundError('Partition directory %s does not exist.' %
              computer_partition_path)
        # Check state of partition. If it is in "destroyed" state, check if it
        # partition is actually installed in the Computer or if it is "free"
830
        # partition, and check if it has some Software information.
831 832 833
        # XXX-Cedric: Temporary AND ugly solution to check if an instance
        # is in the partition. Dangerous because not 100% sure it is empty
        computer_partition_state = computer_partition.getState()
834 835 836 837
        try:
          software_url = computer_partition.getSoftwareRelease().getURI()
        except (NotFoundError, TypeError, NameError):
          software_url = None
838
        if computer_partition_state == COMPUTER_PARTITION_DESTROYED_STATE and \
839 840
           os.listdir(computer_partition_path) == [] and \
           not software_url:
841
          continue
842

843 844 845 846 847 848 849
        # Everything seems fine
        filtered_computer_partition_list.append(computer_partition)

      # XXX-Cedric: factor all this error handling

      # Send log before exiting
      except (SystemExit, KeyboardInterrupt):
850
        computer_partition.error(traceback.format_exc(), logger=self.logger)
851 852 853
        raise

      # Buildout failed: send log but don't print it to output (already done)
Marco Mariani's avatar
Marco Mariani committed
854
      except BuildoutFailedError as exc:
855
        try:
856
          computer_partition.error(exc, logger=self.logger)
857 858 859
        except (SystemExit, KeyboardInterrupt):
          raise
        except Exception:
860 861
          self.logger.error('Problem during reporting error, continuing:\n%s' %
                            traceback.format_exc())
862 863

      # For everything else: log it, send it, continue.
Marco Mariani's avatar
Marco Mariani committed
864
      except Exception as exc:
865
        self.logger.error(traceback.format_exc())
866
        try:
867
          computer_partition.error(exc, logger=self.logger)
868 869 870
        except (SystemExit, KeyboardInterrupt):
          raise
        except Exception:
871 872
          self.logger.error('Problem during reporting error, continuing:\n%s' %
                            traceback.format_exc())
873 874 875 876 877 878 879

    return filtered_computer_partition_list

  def processComputerPartitionList(self):
    """
    Will start supervisord and process each Computer Partition.
    """
880
    self.logger.info('Processing computer partitions...')
881 882 883
    # Prepares environment
    self.checkEnvironmentAndCreateStructure()
    self._launchSupervisord()
884 885

    # Boolean to know if every instance has correctly been deployed
886
    clean_run = True
887 888
    # Boolean to know if every promises correctly passed
    clean_run_promise = True
889 890 891 892 893 894 895 896 897 898

    # Filter all dummy / empty partitions
    computer_partition_list = self.FilterComputerPartitionList(
        self.getComputerPartitionList())

    for computer_partition in computer_partition_list:
      # Nothing should raise outside of the current loop iteration, so that
      # even if something is terribly wrong while processing an instance, it
      # won't prevent processing other ones.
      try:
899
        # Process the partition itself
900
        self.processComputerPartition(computer_partition)
901

902
      # Send log before exiting
Łukasz Nowak's avatar
Łukasz Nowak committed
903
      except (SystemExit, KeyboardInterrupt):
904
        computer_partition.error(traceback.format_exc(), logger=self.logger)
Łukasz Nowak's avatar
Łukasz Nowak committed
905
        raise
906

Marco Mariani's avatar
Marco Mariani committed
907
      except Slapgrid.PromiseError as exc:
908 909
        clean_run_promise = False
        try:
910
          self.logger.error(exc)
911
          computer_partition.error(exc, logger=self.logger)
912 913 914
        except (SystemExit, KeyboardInterrupt):
          raise
        except Exception:
915 916
          self.logger.error('Problem during reporting error, continuing:\n%s' %
                            traceback.format_exc())
917

918
      # Buildout failed: send log but don't print it to output (already done)
Marco Mariani's avatar
Marco Mariani committed
919
      except BuildoutFailedError as exc:
920 921
        clean_run = False
        try:
922
          computer_partition.error(exc, logger=self.logger)
923 924 925
        except (SystemExit, KeyboardInterrupt):
          raise
        except Exception:
926 927
          self.logger.error('Problem during reporting error, continuing:\n%s' %
                            traceback.format_exc())
928 929

      # For everything else: log it, send it, continue.
Marco Mariani's avatar
Marco Mariani committed
930
      except Exception as exc:
Łukasz Nowak's avatar
Łukasz Nowak committed
931
        clean_run = False
932
        self.logger.error(traceback.format_exc())
933
        try:
934
          computer_partition.error(exc, logger=self.logger)
935 936 937
        except (SystemExit, KeyboardInterrupt):
          raise
        except Exception:
938 939
          self.logger.error('Problem during reporting error, continuing:\n%s' %
                            traceback.format_exc())
940

941
    self.logger.info('Finished computer partitions.')
942 943 944 945 946 947 948

    # Return success value
    if not clean_run:
      return SLAPGRID_FAIL
    if not clean_run_promise:
      return SLAPGRID_PROMISE_FAIL
    return SLAPGRID_SUCCESS
Łukasz Nowak's avatar
Łukasz Nowak committed
949

950

951 952 953 954 955
  def validateXML(self, to_be_validated, xsd_model):
    """Validates a given xml file"""
    #We retrieve the xsd model
    xsd_model = StringIO.StringIO(xsd_model)
    xmlschema_doc = etree.parse(xsd_model)
Łukasz Nowak's avatar
Łukasz Nowak committed
956 957
    xmlschema = etree.XMLSchema(xmlschema_doc)

958
    try:
959
      document = etree.fromstring(to_be_validated)
Marco Mariani's avatar
Marco Mariani committed
960
    except (etree.XMLSyntaxError, etree.DocumentInvalid) as exc:
961
      self.logger.info('Failed to parse this XML report :  %s\n%s' % \
Marco Mariani's avatar
Marco Mariani committed
962
        (to_be_validated, _formatXMLError(exc)))
963
      self.logger.error(_formatXMLError(exc))
964 965
      return False

Łukasz Nowak's avatar
Łukasz Nowak committed
966 967 968 969 970
    if xmlschema.validate(document):
      return True

    return False

971 972 973
  def asXML(self, computer_partition_usage_list):
    """Generates a XML report from computer partition usage list
    """
974 975 976 977 978 979 980 981 982 983 984 985 986 987
    xml = ['<?xml version="1.0"?>',
           '<journal>',
           '<transaction type="Sale Packing List">',
           '<title>Resource consumptions</title>',
           '<start_date></start_date>',
           '<stop_date>%s</stop_date>' % time.strftime("%Y-%m-%d at %H:%M:%S"),
           '<reference>%s</reference>' % self.computer_id,
           '<currency></currency>',
           '<payment_mode></payment_mode>',
           '<category></category>',
           '<arrow type="Administration">',
           '<source></source>',
           '<destination></destination>',
           '</arrow>']
988 989 990

    for computer_partition_usage in computer_partition_usage_list:
      try:
991
        root = etree.fromstring(computer_partition_usage.usage)
Marco Mariani's avatar
Marco Mariani committed
992
      except UnicodeError as exc:
993
        self.logger.info("Failed to read %s." % computer_partition_usage.usage)
994
        self.logger.error(UnicodeError)
Marco Mariani's avatar
Marco Mariani committed
995 996
        raise UnicodeError("Failed to read %s: %s" % (computer_partition_usage.usage, exc))
      except (etree.XMLSyntaxError, etree.DocumentInvalid) as exc:
Cédric de Saint Martin's avatar
YATTA  
Cédric de Saint Martin committed
997
        self.logger.info("Failed to parse %s." % (computer_partition_usage.usage))
Marco Mariani's avatar
Marco Mariani committed
998 999 1000 1001
        self.logger.error(exc)
        raise _formatXMLError(exc)
      except Exception as exc:
        raise Exception("Failed to generate XML report: %s" % exc)
1002 1003

      for movement in root.findall('movement'):
1004 1005 1006 1007
        xml.append('<movement>')
        for child in movement.getchildren():
          if child.tag == "reference":
            xml.append('<%s>%s</%s>' % (child.tag, computer_partition_usage.getId(), child.tag))
1008
          else:
1009 1010
            xml.append('<%s>%s</%s>' % (child.tag, child.text, child.tag))
        xml.append('</movement>')
1011

1012
    xml.append('</transaction></journal>')
1013

1014
    return ''.join(xml)
1015

Łukasz Nowak's avatar
Łukasz Nowak committed
1016 1017 1018
  def agregateAndSendUsage(self):
    """Will agregate usage from each Computer Partition.
    """
1019 1020 1021 1022
    # Prepares environment
    self.checkEnvironmentAndCreateStructure()
    self._launchSupervisord()

Łukasz Nowak's avatar
Łukasz Nowak committed
1023 1024
    slap_computer_usage = self.slap.registerComputer(self.computer_id)
    computer_partition_usage_list = []
1025
    self.logger.info('Aggregating and sending usage reports...')
Łukasz Nowak's avatar
Łukasz Nowak committed
1026

1027 1028 1029 1030 1031 1032 1033 1034 1035
    #We retrieve XSD models
    try:
      computer_consumption_model = \
        pkg_resources.resource_string(
          'slapos.slap',
          'doc/computer_consumption.xsd')
    except IOError:
      computer_consumption_model = \
        pkg_resources.resource_string(
1036
          __name__,
1037 1038 1039 1040 1041 1042 1043 1044 1045 1046
          '../../../../slapos/slap/doc/computer_consumption.xsd')

    try:
      partition_consumption_model = \
        pkg_resources.resource_string(
          'slapos.slap',
          'doc/partition_consumption.xsd')
    except IOError:
      partition_consumption_model = \
        pkg_resources.resource_string(
1047
          __name__,
1048 1049
          '../../../../slapos/slap/doc/partition_consumption.xsd')

Łukasz Nowak's avatar
Łukasz Nowak committed
1050
    clean_run = True
1051 1052 1053
    # Loop on the different computer partitions
    computer_partition_list = self.FilterComputerPartitionList(
       slap_computer_usage.getComputerPartitionList())
1054

1055
    for computer_partition in computer_partition_list:
1056 1057
      try:
        computer_partition_id = computer_partition.getId()
1058

1059 1060 1061 1062 1063 1064 1065 1066
        #We want execute all the script in the report folder
        instance_path = os.path.join(self.instance_root,
            computer_partition.getId())
        report_path = os.path.join(instance_path, 'etc', 'report')
        if os.path.isdir(report_path):
          script_list_to_run = os.listdir(report_path)
        else:
          script_list_to_run = []
Marco Mariani's avatar
Marco Mariani committed
1067

1068 1069 1070 1071 1072 1073
        #We now generate the pseudorandom name for the xml file
        # and we add it in the invocation_list
        f = tempfile.NamedTemporaryFile()
        name_xml = '%s.%s' % ('slapreport', os.path.basename(f.name))
        path_to_slapreport = os.path.join(instance_path, 'var', 'xml_report',
            name_xml)
Marco Mariani's avatar
Marco Mariani committed
1074

1075 1076 1077 1078 1079 1080 1081 1082 1083
        failed_script_list = []
        for script in script_list_to_run:
          invocation_list = []
          invocation_list.append(os.path.join(instance_path, 'etc', 'report',
            script))
          #We add the xml_file name in the invocation_list
          #f = tempfile.NamedTemporaryFile()
          #name_xml = '%s.%s' % ('slapreport', os.path.basename(f.name))
          #path_to_slapreport = os.path.join(instance_path, 'var', name_xml)
Marco Mariani's avatar
Marco Mariani committed
1084

1085 1086 1087 1088 1089 1090 1091 1092
          invocation_list.append(path_to_slapreport)
          #Dropping privileges
          uid, gid = None, None
          stat_info = os.stat(instance_path)
          #stat sys call to get statistics informations
          uid = stat_info.st_uid
          gid = stat_info.st_gid
          process_handler = SlapPopen(invocation_list,
1093
                                      preexec_fn=lambda: dropPrivileges(uid, gid, logger=self.logger),
Marco Mariani's avatar
Marco Mariani committed
1094 1095
                                      cwd=os.path.join(instance_path, 'etc', 'report'),
                                      env=None,
Marco Mariani's avatar
Marco Mariani committed
1096
                                      stdout=subprocess.PIPE,
1097 1098
                                      stderr=subprocess.STDOUT,
                                      logger=self.logger)
1099 1100 1101 1102
          if process_handler.returncode is None:
            process_handler.kill()
          if process_handler.returncode != 0:
            clean_run = False
1103
            failed_script_list.append("Script %r failed." % script)
1104
            self.logger.warning('Failed to run %r' % invocation_list)
1105
          if len(failed_script_list):
1106
            computer_partition.error('\n'.join(failed_script_list), logger=self.logger)
1107 1108
      # Whatever happens, don't stop processing other instances
      except Exception:
1109 1110 1111
        self.logger.info('Cannot run usage script(s) for %r: %s' % (
                            computer_partition.getId(),
                            traceback.format_exc()))
Łukasz Nowak's avatar
Łukasz Nowak committed
1112

Cédric de Saint Martin's avatar
Cédric de Saint Martin committed
1113
    #Now we loop through the different computer partitions to report
Łukasz Nowak's avatar
Łukasz Nowak committed
1114
    report_usage_issue_cp_list = []
1115
    for computer_partition in computer_partition_list:
1116 1117 1118 1119 1120 1121 1122 1123 1124
      try:
        filename_delete_list = []
        computer_partition_id = computer_partition.getId()
        instance_path = os.path.join(self.instance_root, computer_partition_id)
        dir_reports = os.path.join(instance_path, 'var', 'xml_report')
        #The directory xml_report contain a number of files equal
        #to the number of software instance running inside the same partition
        if os.path.isdir(dir_reports):
          filename_list = os.listdir(dir_reports)
Łukasz Nowak's avatar
Łukasz Nowak committed
1125
        else:
1126
          filename_list = []
1127
        #self.logger.debug('name List %s' % filename_list)
Marco Mariani's avatar
Marco Mariani committed
1128

1129
        for filename in filename_list:
Marco Mariani's avatar
Marco Mariani committed
1130

1131 1132
          file_path = os.path.join(dir_reports, filename)
          if os.path.exists(file_path):
Marco Mariani's avatar
Marco Mariani committed
1133
            usage = open(file_path, 'r').read()
Marco Mariani's avatar
Marco Mariani committed
1134

1135 1136
            #We check the validity of xml content of each reports
            if not self.validateXML(usage, partition_consumption_model):
1137 1138 1139
              self.logger.info('WARNING: The XML file %s generated by slapreport is '
                               'not valid - This report is left as is at %s where you can '
                               'inspect what went wrong ' % (filename, dir_reports))
1140 1141 1142 1143 1144 1145 1146 1147 1148
              # Warn the SlapOS Master that a partition generates corrupted xml
              # report
            else:
              computer_partition_usage = self.slap.registerComputerPartition(
                      self.computer_id, computer_partition_id)
              computer_partition_usage.setUsage(usage)
              computer_partition_usage_list.append(computer_partition_usage)
              filename_delete_list.append(filename)
          else:
1149
            self.logger.debug('Usage report %r not found, ignored' % file_path)
Łukasz Nowak's avatar
Łukasz Nowak committed
1150

1151 1152 1153 1154 1155 1156
        #After sending the aggregated file we remove all the valid xml reports
        for filename in filename_delete_list:
          os.remove(os.path.join(dir_reports, filename))

      # Whatever happens, don't stop processing other instances
      except Exception:
1157 1158 1159
        self.logger.info('Cannot run usage script(s) for %r: %s' % (
                            computer_partition.getId(),
                            traceback.format_exc()))
1160 1161

    for computer_partition_usage in computer_partition_usage_list:
1162 1163 1164
      self.logger.info('computer_partition_usage_list: %s - %s' % (
                          computer_partition_usage.usage,
                          computer_partition_usage.getId()))
1165 1166 1167

    #If there is, at least, one report
    if computer_partition_usage_list != []:
Łukasz Nowak's avatar
Łukasz Nowak committed
1168
      try:
1169 1170 1171
        #We generate the final XML report with asXML method
        computer_consumption = self.asXML(computer_partition_usage_list)

1172
        self.logger.info('Final xml report: %s' % computer_consumption)
1173 1174 1175

        #We test the XML report before sending it
        if self.validateXML(computer_consumption, computer_consumption_model):
1176
          self.logger.info('XML file generated by asXML is valid')
1177 1178
          slap_computer_usage.reportUsage(computer_consumption)
        else:
1179
          self.logger.info('XML file generated by asXML is not valid !')
1180
          raise ValueError('XML file generated by asXML is not valid !')
Łukasz Nowak's avatar
Łukasz Nowak committed
1181
      except Exception:
1182 1183 1184
        issue = "Cannot report usage for %r: %s" % (
                    computer_partition.getId(),
                    traceback.format_exc())
1185
        self.logger.info(issue)
1186
        computer_partition.error(issue, logger=self.logger)
Łukasz Nowak's avatar
Łukasz Nowak committed
1187 1188
        report_usage_issue_cp_list.append(computer_partition_id)

1189
    for computer_partition in computer_partition_list:
1190
      if computer_partition.getState() == COMPUTER_PARTITION_DESTROYED_STATE:
Łukasz Nowak's avatar
Łukasz Nowak committed
1191
        try:
1192
          computer_partition_id = computer_partition.getId()
1193
          try:
1194
             software_url = computer_partition.getSoftwareRelease().getURI()
Marco Mariani's avatar
Marco Mariani committed
1195
             software_path = os.path.join(self.software_root, md5digest(software_url))
1196 1197 1198
          except (NotFoundError, TypeError):
            software_url = None
            software_path = None
1199

1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213
          local_partition = Partition(
            software_path=software_path,
            instance_path=os.path.join(self.instance_root,
                computer_partition.getId()),
            supervisord_partition_configuration_path=os.path.join(
              self.supervisord_configuration_directory, '%s.conf' %
              computer_partition_id),
            supervisord_socket=self.supervisord_socket,
            computer_partition=computer_partition,
            computer_id=self.computer_id,
            partition_id=computer_partition_id,
            server_url=self.master_url,
            software_release_url=software_url,
            certificate_repository_path=self.certificate_repository_path,
1214
            buildout=self.buildout,
1215
            logger=self.logger)
Łukasz Nowak's avatar
Łukasz Nowak committed
1216 1217 1218 1219
          local_partition.stop()
          try:
            computer_partition.stopped()
          except (SystemExit, KeyboardInterrupt):
1220
            computer_partition.error(traceback.format_exc(), logger=self.logger)
Łukasz Nowak's avatar
Łukasz Nowak committed
1221 1222 1223
            raise
          except Exception:
            pass
1224
          if computer_partition.getId() in report_usage_issue_cp_list:
1225 1226
            self.logger.info('Ignoring destruction of %r, as no report usage was sent' %
                                computer_partition.getId())
1227 1228
            continue
          local_partition.destroy()
Łukasz Nowak's avatar
Łukasz Nowak committed
1229
        except (SystemExit, KeyboardInterrupt):
1230
          computer_partition.error(traceback.format_exc(), logger=self.logger)
Łukasz Nowak's avatar
Łukasz Nowak committed
1231 1232 1233
          raise
        except Exception:
          clean_run = False
Marco Mariani's avatar
Marco Mariani committed
1234
          exc = traceback.format_exc()
1235
          computer_partition.error(exc, logger=self.logger)
1236
          self.logger.error(exc)
Łukasz Nowak's avatar
Łukasz Nowak committed
1237 1238
        try:
          computer_partition.destroyed()
Marco Mariani's avatar
Marco Mariani committed
1239
        except NotFoundError:
1240 1241 1242
          self.logger.debug('Ignored slap error while trying to inform about '
                            'destroying not fully configured Computer Partition %r' %
                                computer_partition.getId())
1243
        except ServerError as server_error:
1244 1245 1246
          self.logger.debug('Ignored server error while trying to inform about '
                            'destroying Computer Partition %r. Error is:\n%r' %
                                (computer_partition.getId(), server_error.args[0]))
Łukasz Nowak's avatar
Łukasz Nowak committed
1247

1248
    self.logger.info('Finished usage reports.')
1249 1250 1251 1252 1253

    # Return success value
    if not clean_run:
      return SLAPGRID_FAIL
    return SLAPGRID_SUCCESS