Merge branch 'collect'

setup.py

@@ -48,6 +48,7 @@ setup(name=name,
           'netifaces', # to fetch information about network devices
           'setuptools', # namespaces
           'supervisor', # slapgrid uses supervisor to manage processes
+          'psutil',
           'xml_marshaller>=0.9.3', # to unmarshall/marshall python objects to/from
                                    # XML
           'zope.interface', # slap library implementes interfaces
@@ -72,17 +73,7 @@ setup(name=name,
       entry_points={
         'console_scripts': [
           'slapos-watchdog = slapos.grid.watchdog:main',
-          'slapproxy = slapos.cli_legacy.proxy_start:main',
           'slapos = slapos.cli.entry:main',
-          # Deprecated entry points
-          'slapconsole = slapos.cli_legacy.console:console',
-          'slapformat = slapos.cli_legacy.format:main',
-          'slapgrid-sr = slapos.cli_legacy.slapgrid:runSoftwareRelease',
-          'slapgrid-cp = slapos.cli_legacy.slapgrid:runComputerPartition',
-          'slapgrid-ur = slapos.cli_legacy.slapgrid:runUsageReport',
-          'slapgrid-supervisorctl = slapos.cli_legacy.svcbackend:supervisorctl',
-          'slapgrid-supervisord = slapos.cli_legacy.svcbackend:supervisord',
-          'bang = slapos.cli_legacy.bang:main',
         ],
         'slapos.cli': [
           # Utilities
@@ -102,6 +93,7 @@ setup(name=name,
           'node software = slapos.cli.slapgrid:SoftwareCommand',
           'node instance = slapos.cli.slapgrid:InstanceCommand',
           'node boot = slapos.cli.boot:BootCommand',
+          'node collect = slapos.cli.collect:CollectCommand',
           # SlapOS client commands
           'console = slapos.cli.console:ConsoleCommand',
           'configure local = slapos.cli.configure_local:ConfigureLocalCommand',

slapos/cli/collect.py

+# -*- coding: utf-8 -*-
+
+import subprocess
+from time import sleep
+import socket
+import glob
+import os
+from slapos.collect import do_collect
+from slapos.cli.command import must_be_root
+from slapos.cli.entry import SlapOSApp
+from slapos.cli.config import ConfigCommand
+
+class CollectCommand(ConfigCommand):
+    """
+    Collect system consumption and data and store.
+    """
+    command_group = 'node'
+
+    def get_parser(self, prog_name):
+        ap = super(CollectCommand, self).get_parser(prog_name)
+        return ap
+
+    @must_be_root
+    def take_action(self, args):
+        configp = self.fetch_config(args)
+        do_collect(configp)

slapos/cli/register.py

@@ -174,10 +174,10 @@ def save_former_config(conf):
 
 
 def fetch_configuration_template():
-    req = requests.get('http://git.erp5.org/gitweb/slapos.core.git/blob_plain/HEAD:/slapos.cfg.example')
-    req.raise_for_status()
-    return req.text
-
+    template_arg_list = (__name__.split('.')[0], 'slapos.cfg.example')
+    with pkg_resources.resource_stream(*template_arg_list) as fout:
+      slapos_node_configuration_template = fout.read()
+    return slapos_node_configuration_template
 
 def slapconfig(conf):
     """Base Function to configure slapos in /etc/opt/slapos"""
@@ -200,13 +200,6 @@ def slapconfig(conf):
         if not dry_run:
             os.mkdir(slap_conf_dir, 0o711)
 
-    # Hack that should be removed when we switch to re6st
-    # Force start of vpn
-    openvpn_needed_file = os.path.join(slap_conf_dir, 'openvpn-needed')
-    if not os.path.exists(openvpn_needed_file):
-        if not dry_run:
-            open(openvpn_needed_file, 'w').write('')
-
     user_certificate_repository_path = os.path.join(slap_conf_dir, 'ssl')
     if not os.path.exists(user_certificate_repository_path):
         conf.logger.info('Creating directory: %s', user_certificate_repository_path)
@@ -344,5 +337,5 @@ def do_register(conf):
     slapconfig(conf)
 
     conf.logger.info('Node has successfully been configured as %s.', COMP)
-    conf.logger.info('Now please invoke /usr/sbin/slapos-start on your site.')
+    conf.logger.info('Now please invoke slapos node boot on your site.')
     return 0

slapos/cli_legacy/bang.py

-# -*- coding: utf-8 -*-
-##############################################################################
-#
-# Copyright (c) 2011, 2012 Vifib SARL and Contributors.
-# All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly advised to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-
-import argparse
-import ConfigParser
-
-from slapos.bang import do_bang
-
-
-def main(*args):
-    ap = argparse.ArgumentParser()
-    ap.add_argument('-m', '--message', default='', help='Message for bang.')
-    ap.add_argument('configuration_file', type=argparse.FileType(),
-                    help='SlapOS configuration file.')
-    if args:
-        args = ap.parse_args(list(args))
-    else:
-        args = ap.parse_args()
-    configp = ConfigParser.SafeConfigParser()
-    configp.readfp(args.configuration_file)
-    do_bang(configp, args.message)

slapos/cli_legacy/console.py

-# -*- coding: utf-8 -*-
-
-import argparse
-import os
-import textwrap
-
-from slapos.client import ClientConfig, init, do_console
-from slapos.cli_legacy.util import get_config_parser
-
-
-def console():
-  description = textwrap.dedent("""\
-    slapconsole allows you interact with slap API. You can play with the global
-    "slap" object and with the global "request" method.
-
-    examples :
-    >>> # Request instance
-    >>> request(kvm, "myuniquekvm")
-    >>> # Request software installation on owned computer
-    >>> supply(kvm, "mycomputer")
-    >>> # Fetch instance informations on already launched instance
-    >>> request(kvm, "myuniquekvm").getConnectionParameter("url")""")
-
-  ap = argparse.ArgumentParser(description=description,
-                               formatter_class=argparse.RawDescriptionHelpFormatter)
-
-  ap.add_argument('-u', '--master_url',
-                  default=None,
-                  help='Url of SlapOS Master to use.')
-
-  ap.add_argument('-k', '--key_file',
-                  help="SSL Authorisation key file.")
-
-  ap.add_argument('-c', '--cert_file',
-                  help="SSL Authorisation certificate file.")
-
-  ap.add_argument('configuration_file',
-                  help='path to slapos.cfg')
-
-  args = ap.parse_args()
-
-  if not os.path.isfile(args.configuration_file):
-    ap.error("%s: Not found or not a regular file." % args.configuration_file)
-
-  configp = get_config_parser(args.configuration_file)
-  conf = ClientConfig(args, configp)
-  local = init(conf)
-  do_console(local)

slapos/cli_legacy/format.py

-# -*- coding: utf-8 -*-
-##############################################################################
-#
-# Copyright (c) 2010, 2011, 2012 Vifib SARL and Contributors.
-# All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly advised to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-
-import argparse
-import ConfigParser
-import logging
-import sys
-import os
-
-from slapos.format import FormatConfig, UsageError, tracing_monkeypatch, do_format
-
-
-def main(*args):
-  "Run default configuration."
-
-  ap = argparse.ArgumentParser()
-
-  ap.add_argument('-x', '--computer_xml',
-                  help="Path to file with computer's XML. If does not exists, will be created",
-                  default=None)
-
-  ap.add_argument('--computer_json',
-                  help="Path to a JSON version of the computer's XML (for development only).",
-                  default=None)
-
-  ap.add_argument('-l', '--log_file',
-                  help="The path to the log file used by the script.")
-
-  ap.add_argument('-i', '--input_definition_file',
-                  help="Path to file to read definition of computer instead of "
-                  "declaration. Using definition file allows to disable "
-                  "'discovery' of machine services and allows to define computer "
-                  "configuration in fully controlled manner.")
-
-  ap.add_argument('-o', '--output_definition_file',
-                  help="Path to file to write definition of computer from "
-                  "declaration.")
-
-  ap.add_argument('-n', '--dry_run',
-                  help="Don't actually do anything.",
-                  default=False,
-                  action="store_true")
-
-  ap.add_argument('-v', '--verbose',
-                  default=False,
-                  action="store_true",
-                  help="Verbose output.")
-
-  # the console option is actually ignored and not used anymore.
-  ap.add_argument('-c', '--console',
-                  default=False,
-                  action="store_true",
-                  help="Console output.")
-
-  ap.add_argument('--alter_user',
-                  choices=['True', 'False'],
-                  help="Shall slapformat alter user database [default: True]")
-
-  ap.add_argument('--alter_network',
-                  choices=['True', 'False'],
-                  help="Shall slapformat alter network configuration [default: True]")
-
-  ap.add_argument('--now',
-                  help="Launch slapformat without delay",
-                  default=False,
-                  action="store_true")
-
-  ap.add_argument('configuration_file',
-                  help='path to slapos.cfg')
-
-  if args:
-    args = ap.parse_args(list(args))
-  else:
-    args = ap.parse_args()
-
-  logger = logging.getLogger("slapformat")
-  logger.addHandler(logging.StreamHandler())
-
-  if args.verbose:
-    logger.setLevel(logging.DEBUG)
-    logger.debug("Verbose mode enabled.")
-  else:
-    logger.setLevel(logging.INFO)
-
-  conf = FormatConfig(logger=logger)
-
-  configp = ConfigParser.SafeConfigParser()
-  if configp.read(args.configuration_file) != [args.configuration_file]:
-    raise UsageError('Cannot find or parse configuration file: %s' % args.configuration_file)
-
-  conf.mergeConfig(args, configp)
-
-  if conf.log_file:
-    if not os.path.isdir(os.path.dirname(conf.log_file)):
-      # fallback to console only if directory for logs does not exists and
-      # continue to run
-      raise ValueError('Please create directory %r to store %r log file' % (
-        os.path.dirname(conf.log_file), conf.log_file))
-    else:
-      file_handler = logging.FileHandler(conf.log_file)
-      file_handler.setFormatter(logging.Formatter("%(asctime)s - "
-        "%(name)s - %(levelname)s - %(message)s"))
-      conf.logger.addHandler(file_handler)
-      conf.logger.info('Configured logging to file %r' % conf.log_file)
-
-  try:
-    conf.setConfig()
-  except UsageError as exc:
-    sys.stderr.write(exc.message + '\n')
-    sys.stderr.write("For help use --help\n")
-    sys.exit(1)
-
-  tracing_monkeypatch(conf)
-
-  try:
-    do_format(conf=conf)
-  except:
-    conf.logger.exception('Uncaught exception:')
-    raise

slapos/cli_legacy/proxy_start.py

-# -*- coding: utf-8 -*-
-# vim: set et sts=2:
-
-import argparse
-import ConfigParser
-import logging
-import os
-import sys
-
-from slapos.proxy import ProxyConfig, do_proxy
-
-class UsageError(Exception):
-  pass
-
-
-def main():
-  ap = argparse.ArgumentParser()
-
-  ap.add_argument('-l', '--log_file',
-                  help='The path to the log file used by the script.')
-
-  ap.add_argument('-v', '--verbose',
-                  action='store_true',
-                  help='Verbose output.')
-
-  # XXX not used anymore, deprecated
-  ap.add_argument('-c', '--console',
-                  action='store_true',
-                  help='Console output.')
-
-  ap.add_argument('-u', '--database-uri',
-                  help='URI for sqlite database')
-
-  ap.add_argument('configuration_file',
-                  help='path to slapos.cfg')
-
-  args = ap.parse_args()
-
-  logger = logging.getLogger('slapproxy')
-  logger.addHandler(logging.StreamHandler())
-
-  if args.verbose:
-    logger.setLevel(logging.DEBUG)
-  else:
-    logger.setLevel(logging.INFO)
-
-  conf = ProxyConfig(logger=logger)
-
-  configp = ConfigParser.SafeConfigParser()
-  if configp.read(args.configuration_file) != [args.configuration_file]:
-    raise UsageError('Cannot find or parse configuration file: %s' % args.configuration_file)
-
-  conf.mergeConfig(args, configp)
-
-  if conf.log_file:
-    if not os.path.isdir(os.path.dirname(conf.log_file)):
-      raise ValueError('Please create directory %r to store %r log file' % (
-        os.path.dirname(conf.log_file), conf.log_file))
-    file_handler = logging.FileHandler(conf.log_file)
-    file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
-    logger.addHandler(file_handler)
-    logger.info('Configured logging to file %r' % conf.log_file)
-
-  conf.setConfig()
-
-  try:
-    do_proxy(conf=conf)
-    return_code = 0
-  except SystemExit as err:
-    return_code = err
-
-  sys.exit(return_code)

slapos/cli_legacy/slapgrid.py

-# -*- coding: utf-8 -*-
-# vim: set et sts=2:
-
-import argparse
-import ConfigParser
-import logging
-import sys
-
-from slapos.grid.utils import setRunning, setFinished
-from slapos.grid.slapgrid import (merged_options, check_missing_parameters,
-                                  check_missing_files, random_delay, create_slapgrid_object)
-
-
-def parse_arguments(*argument_tuple):
-  """Parse arguments and return options dictionary merged with the config file."""
-
-  ap = argparse.ArgumentParser()
-  ap.add_argument('--instance-root',
-                  help='The instance root directory location.')
-  ap.add_argument('--software-root',
-                  help='The software_root directory location.')
-  ap.add_argument('--master-url',
-                  help='The master server URL. Mandatory.')
-  ap.add_argument('--computer-id',
-                  help='The computer id defined in the server.')
-  ap.add_argument('--supervisord-socket',
-                  help='The socket supervisor will use.')
-  ap.add_argument('--supervisord-configuration-path',
-                  help='The location where supervisord configuration will be stored.')
-  ap.add_argument('--buildout', default=None,
-                  help='Location of buildout binary.')
-  ap.add_argument('--pidfile',
-                  help='The location where pidfile will be created.')
-  ap.add_argument('--logfile',
-                  help='The location where slapgrid logfile will be created.')
-  ap.add_argument('--key_file',
-                  help='SSL Authorisation key file.')
-  ap.add_argument('--cert_file',
-                  help='SSL Authorisation certificate file.')
-  ap.add_argument('--signature_private_key_file',
-                  help='Signature private key file.')
-  ap.add_argument('--master_ca_file',
-                  help='Root certificate of SlapOS master key.')
-  ap.add_argument('--certificate_repository_path',
-                  help='Path to directory where downloaded certificates would be stored.')
-  ap.add_argument('-v', '--verbose', action='store_true',
-                  help='Be verbose.')
-  ap.add_argument('--maximum-periodicity', type=int, default=None,
-                  help='Periodicity at which buildout should be run in instance.')
-  ap.add_argument('--promise-timeout', type=int, default=3,
-                  help='Promise timeout in seconds.')
-  ap.add_argument('--now', action='store_true',
-                  help='Launch slapgrid without delay. Default behavior.')
-  ap.add_argument('--all', action='store_true',
-                  help='Launch slapgrid to process all Software Releases '
-                       'and/or Computer Partitions.')
-  ap.add_argument('--only-sr',
-                  help='Force the update of a single software release (use url hash), '
-                       'even if is already installed. This option will make all other '
-                       'sofware releases be ignored.')
-  ap.add_argument('--only-cp',
-                  help='Update a single or a list of computer partitions '
-                       '(ie.:slappartX, slappartY), '
-                       'this option will make all other computer partitions be ignored.')
-
-  ap.add_argument('configuration_file', type=argparse.FileType(),
-                  help='SlapOS configuration file.')
-
-  # Deprecated options
-  ap.add_argument('-c', '--console', action='store_true',
-                  help="Deprecated, doesn't do anything.")
-  ap.add_argument('--develop', action='store_true',
-                  help='Deprecated, same as --all.')
-  ap.add_argument('--only_sr',
-                  help='Deprecated, same as --only-sr.')
-  ap.add_argument('--only_cp',
-                  help='Deprecated, same as --only-cp.')
-  ap.add_argument('--maximal_delay',
-                  help='Deprecated. Will only work from configuration file in the future.')
-
-  if not argument_tuple:
-    args = ap.parse_args()
-  else:
-    args = ap.parse_args(list(argument_tuple))
-
-  return args
-
-
-def setup_logger(options):
-  logger = logging.getLogger(__name__)
-
-  if options.get('logfile'):
-    handler = logging.FileHandler(options['logfile'])
-  else:
-    handler = logging.StreamHandler()
-
-  if options['verbose']:
-    handler.setLevel(logging.DEBUG)
-  else:
-    handler.setLevel(logging.INFO)
-
-  formatter = logging.Formatter(fmt='%(asctime)s %(name)-18s: '
-                                '%(levelname)-8s %(message)s',
-                                datefmt='%Y-%m-%dT%H:%M:%S')
-
-  handler.setFormatter(formatter)
-  logger.addHandler(handler)
-
-  return logger
-
-
-def parseArgumentTupleAndReturnSlapgridObject(*argument_tuple):
-  """Returns a new instance of slapgrid.Slapgrid created with argument+config parameters.
-     Also returns the pidfile path, and configures logger.
-  """
-  args = parse_arguments(*argument_tuple)
-
-  configp = ConfigParser.SafeConfigParser()
-  configp.readfp(args.configuration_file)
-
-  options = merged_options(args, configp)
-
-  logger = setup_logger(options)
-
-  check_missing_parameters(options)
-  check_missing_files(options)
-
-  random_delay(options, logger=logger)
-
-  slapgrid_object = create_slapgrid_object(options, logger=logger)
-
-  return slapgrid_object, options.get('pidfile')
-
-
-def realRun(argument_tuple, method):
-  slapgrid_object, pidfile = parseArgumentTupleAndReturnSlapgridObject(*argument_tuple)
-  if pidfile:
-    setRunning(logger=slapgrid_object.logger, pidfile=pidfile)
-  try:
-    return getattr(slapgrid_object, method)()
-  finally:
-    if pidfile:
-      setFinished(pidfile)
-
-
-def runSoftwareRelease(*argument_tuple):
-  """Hook for entry point to process Software Releases"""
-  sys.exit(realRun(argument_tuple, 'processSoftwareReleaseList'))
-
-
-def runComputerPartition(*argument_tuple):
-  """Hook for entry point to process Computer Partitions"""
-  sys.exit(realRun(argument_tuple, 'processComputerPartitionList'))
-
-
-def runUsageReport(*argument_tuple):
-  """Hook for entry point to process Usage Reports"""
-  sys.exit(realRun(argument_tuple, 'agregateAndSendUsage'))

slapos/cli_legacy/svcbackend.py

-# -*- coding: utf-8 -*-
-# vim: set et sts=2:
-##############################################################################
-#
-# Copyright (c) 2010, 2011, 2012 Vifib SARL and Contributors.
-# All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly advised to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-
-import logging
-import os
-from optparse import OptionParser
-import ConfigParser
-
-from slapos.grid.svcbackend import launchSupervisord
-
-
-def getOptionDict(*argument_tuple):
-  usage = """
-Typical usage:
- * %prog CONFIGURATION_FILE [arguments passed to supervisor]
-
-""".strip()
-
-  parser = OptionParser(usage=usage)
-
-  # Parses arguments
-  if argument_tuple:
-    (argument_option_instance, argument_list) = parser.parse_args(list(argument_tuple))
-  else:
-    # No arguments given to entry point : we parse sys.argv.
-    (argument_option_instance, argument_list) = parser.parse_args()
-
-  if not argument_list:
-    parser.error("Configuration file is obligatory. Consult documentation by calling with -h.")
-  configuration_file = argument_list[0]
-  if not os.path.exists(configuration_file):
-    parser.error("Could not read configuration file : %s" % configuration_file)
-
-  slapgrid_configuration = ConfigParser.SafeConfigParser()
-  slapgrid_configuration.read(configuration_file)
-
-  # Merges the two dictionnaries
-  option_dict = dict(slapgrid_configuration.items("slapos"))
-  # Supervisord configuration location
-  option_dict.setdefault('supervisord_configuration_path',
-                         os.path.join(option_dict['instance_root'], 'etc', 'supervisord.conf'))
-  # Supervisord socket
-  option_dict.setdefault('supervisord_socket',
-                         os.path.join(option_dict['instance_root'], 'supervisord.socket'))
-  return option_dict, argument_list[1:]
-
-
-def supervisorctl(*argument_tuple):
-  logger = logging.getLogger('SVCBackend')
-  logger.setLevel(logging.INFO)
-  handler = logging.StreamHandler()
-  logger.addHandler(handler)
-
-  option_dict, args = getOptionDict(*argument_tuple)
-  import supervisor.supervisorctl
-  launchSupervisord(socket=option_dict['supervisord_socket'],
-                    configuration_file=option_dict['supervisord_configuration_path'],
-                    logger=logger)
-  supervisor.supervisorctl.main(args=['-c', option_dict['supervisord_configuration_path']] + args)
-
-
-def supervisord(*argument_tuple):
-  logger = logging.getLogger('SVCBackend')
-  logger.setLevel(logging.INFO)
-  handler = logging.StreamHandler()
-  logger.addHandler(handler)
-
-  option_dict, _ = getOptionDict(*argument_tuple)
-  launchSupervisord(socket=option_dict['supervisord_socket'],
-                    configuration_file=option_dict['supervisord_configuration_path'],
-                    logger=logger)

slapos/cli_legacy/util.py

-# -*- coding: utf-8 -*-
-
-import ConfigParser
-import os
-
-
-def get_config_parser(path):
-    configp = ConfigParser.SafeConfigParser()
-    path = os.path.expanduser(path)
-    if not os.path.isfile(path):
-        raise OSError('Specified configuration file %s does not exist. Exiting.' % path)
-    configp.read(path)
-    return configp

slapos/collect/__init__.py

+from psutil import process_iter, NoSuchProcess, AccessDenied
+from time import time, sleep, strftime
+from slapos.collect.db import Database
+from slapos.util import mkdir_p
+# Local import
+from snapshot import ProcessSnapshot, SystemSnapshot, ComputerSnapshot
+from slapos.collect.reporter import SystemJSONReporterDumper, \
+                                    RawCSVDumper, \
+                                    SystemCSVReporterDumper
+
+from entity import get_user_list, Computer
+
+def _get_time():
+  return strftime("%Y-%m-%d -- %H:%M:%S").split(" -- ")
+
+def build_snapshot(proc):
+  try:
+    return ProcessSnapshot(proc)
+  except NoSuchProcess:
+    return None
+
+def current_state(user_dict):
+  """
+  Iterator used to apply build_snapshot(...) on every single relevant process.
+  A process is considered relevant if its user matches our user list, i.e.
+  its user is a slapos user
+  """
+  process_list = [p for p in process_iter() if p.username() in user_dict]
+  for i, process in enumerate(process_list):
+    yield build_snapshot(process)
+
+def do_collect(conf):
+  """
+  Main function
+  The idea here is to poll system every so many seconds
+  For each poll, we get a list of Snapshots, holding informations about
+  processes. We iterate over that list to store datas on a per user basis:
+    Each user object is a dict, indexed on timestamp. We add every snapshot
+    matching the user so that we get informations for each users
+  """
+  try:
+    collected_date, collected_time = _get_time()
+    user_dict = get_user_list(conf)
+    try:
+      for snapshot in current_state(user_dict):
+        if snapshot:
+          user_dict[snapshot.username].append(snapshot)
+    except (KeyboardInterrupt, SystemExit, NoSuchProcess):
+      raise
+      
+    log_directory = "%s/var/data-log" % conf.get("slapos", "instance_root")
+    mkdir_p(log_directory)
+    database = Database(log_directory)
+
+    computer = Computer(ComputerSnapshot())
+    computer.save(database, collected_date, collected_time)
+
+    for user in user_dict.values():
+      user.save(database, collected_date, collected_time)
+    
+    SystemCSVReporterDumper(database).dump(log_directory)
+    RawCSVDumper(database).dump(log_directory)
+
+  except AccessDenied:
+    print "You HAVE TO execute this script with root permission."
+

slapos/collect/entity.py

+
+def get_user_list(config):
+  nb_user = int(config.get("slapformat", "partition_amount"))
+  name_prefix = config.get("slapformat", "user_base_name")
+  path_prefix = config.get("slapformat", "partition_base_name")
+  instance_root = config.get("slapos", "instance_root")
+  user_dict = {name: User(name, path)
+      for name, path in [
+          (
+            "%s%s" % (name_prefix, nb),
+            "%s/%s%s" % (instance_root, path_prefix, nb)
+          ) for nb in range(nb_user)
+        ]
+      }
+
+  #user_dict['root'] = User("root", "/opt/slapgrid")
+  return user_dict
+
+class User(object):
+  def __init__(self, name, path):
+    self.name = str(name)
+    self.path = str(path)
+    self.snapshot_list = []
+
+  def append(self, value):
+    self.snapshot_list.append(value)
+
+  def save(self, database, collected_date, collected_time):
+    """ Insert collected data on user collector """
+    database.connect()
+    for snapshot_item in self.snapshot_list:
+      database.insertUserSnapshot(self.name,
+            pid=snapshot_item.get("pid"),
+            process=snapshot_item.get("process"),
+            cpu_percent=snapshot_item.get("cpu_percent"),
+            cpu_time=snapshot_item.get("cpu_time"),
+            cpu_num_threads=snapshot_item.get("cpu_num_threads"),
+            memory_percent=snapshot_item.get("memory_percent"),
+            memory_rss=snapshot_item.get("memory_rss"),
+            io_rw_counter=snapshot_item.get("io_rw_counter"),
+            io_cycles_counter=snapshot_item.get("io_cycles_counter"),
+            insertion_date=collected_date, 
+            insertion_time=collected_time)
+    database.commit()
+    database.close()
+
+class Computer(dict):
+
+  def __init__(self, computer_snapshot):
+    self.computer_snapshot = computer_snapshot
+
+  def save(self, database, collected_date, collected_time):
+    database.connect()
+    self._save_computer_snapshot(database, collected_date, collected_time)
+    self._save_system_snapshot(database, collected_date, collected_time)
+    self._save_disk_partition_snapshot(database, collected_date, collected_time)
+    database.commit()
+    database.close()
+
+  def _save_computer_snapshot(self, database, collected_date, collected_time):
+    partition_list = ";".join(["%s=%s" % (x,y) for x,y in \
+                                  self.computer_snapshot.get("partition_list")])
+    database.insertComputerSnapshot(
+            cpu_num_core=self.computer_snapshot.get("cpu_num_core"), 
+            cpu_frequency=self.computer_snapshot.get("cpu_frequency"),
+            cpu_type=self.computer_snapshot.get("cpu_type"),
+            memory_size=self.computer_snapshot.get("memory_size"),
+            memory_type=self.computer_snapshot.get("memory_type"), 
+            partition_list=partition_list,
+            insertion_date=collected_date, 
+            insertion_time=collected_time)
+
+  def _save_system_snapshot(self, database, collected_date, collected_time):
+    snapshot = self.computer_snapshot.get("system_snapshot")
+    database.insertSystemSnapshot(
+      loadavg=snapshot.get("load"),
+      cpu_percent=snapshot.get("cpu_percent"), 
+      memory_used=snapshot.get("memory_used"), 
+      memory_free=snapshot.get("memory_free"),
+      net_in_bytes=snapshot.get("net_in_bytes"),
+      net_in_errors=snapshot.get("net_in_errors"),
+      net_in_dropped=snapshot.get("net_in_dropped"),
+      net_out_bytes=snapshot.get("net_out_bytes"),
+      net_out_errors= snapshot.get("net_out_errors"),
+      net_out_dropped=snapshot.get("net_out_dropped"),
+      insertion_date=collected_date, 
+      insertion_time=collected_time)
+
+  def _save_disk_partition_snapshot(self, database, collected_date, collected_time):
+    for disk_partition in self.computer_snapshot.get("disk_snapshot_list"):
+      database.insertDiskPartitionSnapshot(
+         partition=disk_partition.partition,
+         used=disk_partition.disk_size_used,
+         free=disk_partition.disk_size_free,
+         mountpoint=';'.join(disk_partition.mountpoint_list),
+         insertion_date=collected_date, 
+         insertion_time=collected_time)
+

slapos/collect/reporter.py

+#!/usr/bin/env python
+
+from slapos.collect.db import Database
+from slapos.util import mkdir_p
+import os.path
+import json
+import csv
+from time import strftime
+
+class Dumper(object):
+
+  def __init__(self, database):
+    self.db = database
+
+class SystemReporter(Dumper):
+  
+  def dump(self, folder):
+    """ Dump data """
+    _date = strftime("%Y-%m-%d")
+    self.db.connect()
+    for item, collected_item_list in self.db.exportSystemAsDict(_date).iteritems():
+      self.writeFile(item, folder, collected_item_list)
+
+    for partition, collected_item_list in self.db.exportDiskAsDict(_date).iteritems():
+      partition_id = partition.split("-")[0].split("/")[-1]
+      item = "memory_%s" % partition.split("-")[1]
+      self.writeFile("disk_%s_%s" % (item, partition_id), folder, collected_item_list)
+
+    self.db.close()
+
+class SystemJSONReporterDumper(SystemReporter):
+
+  def writeFile(self, name, folder, collected_entry_list=[]):
+    """ Dump data as json """
+    file_io = open(os.path.join(folder, "system_%s.json" % name), "w")
+    json.dump(collected_entry_list, file_io, sort_keys=True, indent=2)
+    file_io.close()
+
+class SystemCSVReporterDumper(SystemReporter):
+
+  def writeFile(self, name, folder, collected_entry_list=[]):
+    """ Dump data as json """
+    file_io = open(os.path.join(folder, "system_%s.csv" % name), "w")
+    csv_output = csv.writer(file_io)
+    csv_output.writerow(["time", "entry"])
+    for collected_entry in collected_entry_list:
+      csv_output.writerow([collected_entry["time"], collected_entry["entry"]])
+    file_io.close()
+
+class RawDumper(Dumper):
+  """ Dump raw data in a certain format
+  """
+  def dump(self, folder):
+    date = strftime("%Y-%m-%d")
+    self.db.connect()
+    table_list = self.db.getTableList()
+    for date_scope, amount in self.db.getDateScopeList(ignore_date=date):
+      for table in table_list:
+        self.writeFile(table, folder, date_scope, 
+              self.db.select(table, date_scope))
+
+      self.db.markDayAsReported(date_scope, 
+                                table_list=table_list)
+    self.db.commit()
+    self.db.close()
+
+class RawCSVDumper(RawDumper):
+  
+  def writeFile(self, name, folder, date_scope, rows):
+    mkdir_p(os.path.join(folder, date_scope))
+    file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)), "w")
+    csv_output = csv.writer(file_io)
+    csv_output.writerows(rows)
+    file_io.close()
+

slapos/collect/snapshot.py

+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2010, 2011, 2012 Vifib SARL and Contributors.
+# All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public License
+# as published by the Free Software Foundation; either version 2.1
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import psutil
+import os
+
+class _Snapshot(object):
+  def get(self, property, default=None):
+    return getattr(self, property, default)
+
+class ProcessSnapshot(_Snapshot):
+  """ Take a snapshot from the running process
+  """
+  def __init__(self, process=None):
+    assert type(process) is psutil.Process
+    ui_counter_list = process.get_io_counters()
+    self.username = process.username()
+    self.pid = process.pid 
+    # Save full command line from the process.
+    self.name = "%s-%s" % (process.pid, process.create_time())
+    # CPU percentage, we will have to get actual absolute value
+    self.cpu_percent = process.get_cpu_percent(None)
+    # CPU Time
+    self.cpu_time = sum(process.get_cpu_times())
+    # Thread number, might not be really relevant
+    self.cpu_num_threads = process.get_num_threads()      
+    # Memory percentage
+    self.memory_percent = process.get_memory_percent()
+    # Resident Set Size, virtual memory size is not accouned for
+    self.memory_rss = process.get_memory_info()[0]
+    # Byte count, Read and write. OSX NOT SUPPORTED
+    self.io_rw_counter = ui_counter_list[2] + ui_counter_list[3]
+    # Read + write IO cycles
+    self.io_cycles_counter  = ui_counter_list[0] + ui_counter_list[1]
+
+class SystemSnapshot(_Snapshot):
+  """ Take a snapshot from current system usage
+  """
+  def __init__(self):
+    memory = psutil.phymem_usage()
+    net_io = psutil.net_io_counters()
+    
+    self.memory_used = memory.used
+    self.memory_free = memory.free
+    self.memory_percent = memory.percent
+    self.cpu_percent = psutil.cpu_percent()
+    self.load = os.getloadavg()[0]
+    self.net_in_bytes = net_io.bytes_recv
+    self.net_in_errors = net_io.errin
+    self.net_in_dropped = net_io.dropin
+    self.net_out_bytes = net_io.bytes_sent
+    self.net_out_errors = net_io.errout
+    self.net_out_dropped = net_io.dropout
+
+class DiskPartitionSnapshot(_Snapshot):
+  """ Take Snapshot from general disk partitions 
+      usage 
+  """
+  def __init__(self, partition, mountpoint):
+    self.partition = partition
+    self.mountpoint_list = [ mountpoint ]
+    disk = psutil.disk_usage(mountpoint)
+    disk_io = psutil.disk_io_counters()
+
+    self.disk_size_used = disk.used
+    self.disk_size_free = disk.free
+    self.disk_size_percent = disk.percent
+
+class ComputerSnapshot(_Snapshot):
+  """ Take a snapshot from computer informations
+  """
+  def __init__(self):
+    self.cpu_num_core = psutil.NUM_CPUS
+    self.cpu_frequency = 0
+    self.cpu_type = 0
+    self.memory_size = psutil.TOTAL_PHYMEM  
+    self.memory_type = 0
+
+    #
+    # Include a SystemSnapshot and a list DiskPartitionSnapshot
+    # on a Computer Snapshot
+    #
+    self.system_snapshot = SystemSnapshot()
+    self.disk_snapshot_list = []
+    self.partition_list = self._get_physical_disk_info()
+
+  def _get_physical_disk_info(self):
+    partition_dict = {}
+
+    for partition in psutil.disk_partitions():
+      if partition.device not in partition_dict:
+        usage = psutil.disk_usage(partition.mountpoint)
+        partition_dict[partition.device] = usage.total
+        self.disk_snapshot_list.append(
+          DiskPartitionSnapshot(partition.device, 
+                                partition.mountpoint))
+
+    return [(k, v) for k, v in partition_dict.iteritems()]
+