Import erp5.utils.{benchmark,test_browser} from svn.erp5.org:public/erp5/trunk/utils

CHANGES.erp5.util.txt

@@ -4,7 +4,12 @@ Changes
 0.2 (unreleased)
 ----------------
 
- * No changes yet.
+ * Imported from https://svn.erp5.org/repos/public/erp5/trunk/utils/
+
+   - erp5.util.test_browser:
+     Programmable browser for functional and performance tests for ERP5
+   - erp5.util.benchmark:
+     Performance benchmarks for ERP5 with erp5.utils.test_browser
 
 0.1 (2011-08-08)
 ----------------

erp5/util/README.test_browser.txt

+API Documentation
+-----------------
+
+You can generate the API documentation using ``epydoc'':
+
+$ epydoc src/erp5

erp5/util/benchmark/argument.py

+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import argparse
+import functools
+
+class ArgumentType(object):
+  @classmethod
+  def directoryType(cls, path):
+    if not (os.path.isdir(path) and os.access(path, os.W_OK)):
+      raise argparse.ArgumentTypeError("'%s' is not a valid directory or is "\
+                                         "not writable" % path)
+
+    return path
+
+  @classmethod
+  def objectFromModule(cls, module_name, object_name=None,
+                       callable_object=False):
+    if module_name.endswith('.py'):
+      module_name = module_name[:-3]
+
+    if not object_name:
+      object_name = module_name
+
+    import sys
+    sys.path.append(os.getcwd())
+
+    try:
+      module = __import__(module_name, globals(), locals(), [object_name], -1)
+    except Exception, e:
+      raise argparse.ArgumentTypeError("Cannot import '%s.%s': %s" % \
+                                         (module_name, object_name, str(e)))
+
+    try:
+      obj = getattr(module, object_name)
+    except AttributeError:
+      raise argparse.ArgumentTypeError("Could not get '%s' in '%s'" % \
+                                         (object_name, module_name))
+
+    if callable_object and not callable(obj):
+      raise argparse.ArgumentTypeError(
+        "'%s.%s' is not callable" % (module_name, object_name))
+
+    return obj
+
+  @classmethod
+  def strictlyPositiveIntType(cls, value):
+    try:
+      converted_value = int(value)
+    except ValueError:
+      pass
+    else:
+      if converted_value > 0:
+        return converted_value
+
+    raise argparse.ArgumentTypeError('expects a strictly positive integer')
+
+  @classmethod
+  def strictlyPositiveIntOrRangeType(cls, value):
+    try:
+      return cls.strictlyPositiveIntType(value)
+    except argparse.ArgumentTypeError:
+      try:
+        min_max_list = value.split(',')
+      except ValueError:
+        pass
+      else:
+        if len(min_max_list) == 2:
+          minimum, maximum = cls.strictlyPositiveIntType(min_max_list[0]), \
+              cls.strictlyPositiveIntType(min_max_list[1])
+
+          if minimum >= maximum:
+            raise argparse.ArgumentTypeError('%d >= %d' % (minimum, maximum))
+
+          return (minimum, maximum)
+
+    raise argparse.ArgumentTypeError(
+      'expects either a strictly positive integer or a range of strictly '
+      'positive integer separated by a comma')
+
+  @classmethod
+  def ERP5UrlType(cls, url):
+    if url[-1] == '/':
+      url_list = url.rsplit('/', 2)[:-1]
+    else:
+      url_list = url.rsplit('/', 1)
+
+    url_list[0] = url_list[0] + '/'
+    if len(url_list) != 2:
+      raise argparse.ArgumentTypeError("Invalid URL given")
+
+    return url_list

erp5/util/benchmark/examples/createPerson.py

+# -*- coding: utf-8 -*-
+
+def createPerson(result, browser):
+  """
+  Create a Person and add a  telephone number.  It can be ran infinitely (e.g.
+  until it  is interrupted by  the end user)  with 1 concurrent  user, through
+  performance_tester_erp5 with the following command:
+
+  performance_tester_erp5 http://foo.bar:4242/erp5/ 1 createPerson
+
+  Please note that  you must run this command from the  same directory of this
+  script  and userInfo.py.  Further information  about performance_tester_erp5
+  options and arguments are available by specifying ``--help''.
+  """
+  # Go to Persons module (person_module)
+  result('Go to person module',
+         browser.mainForm.submitSelectModule(value='/person_module'))
+
+  # Create a new person and record the time elapsed in seconds
+  result('Add Person', browser.mainForm.submitNew())
+
+  # Check whether it has been successfully created
+  assert browser.getTransitionMessage() == 'Object created.'
+
+  # Fill the first and last name of the newly created person
+  browser.mainForm.getControl(name='field_my_first_name').value = 'Create'
+  browser.mainForm.getControl(name='field_my_last_name').value = 'Person'
+
+  # Submit the changes, record the time elapsed in seconds
+  result('Save', browser.mainForm.submitSave())
+
+  # Check whether the changes have been successfully updated
+  assert browser.getTransitionMessage() == 'Data updated.'
+  person_url = browser.url
+
+  # Add phone number
+  result('Add telephone',
+         browser.mainForm.submitSelectAction(value='add Telephone'))
+
+  # Fill telephone title and number
+  browser.mainForm.getControl(name='field_my_title'). value = 'Personal'
+  browser.mainForm.getControl(name='field_my_telephone_number').value = '0123456789'
+
+  # Submit the changes, record the time elapsed in seconds
+  result('Save', browser.mainForm.submitSave())
+
+  # Check whether the changes have been successfully updated
+  assert browser.getTransitionMessage() == 'Data updated.'
+
+  # Go back to the Person page before validating
+  browser.open(person_url)
+
+  # Validate it (as the workflow action may not be available yet, try 5 times
+  # and sleep 5s between each attempts before failing)
+  show_validate_time, waiting_for_validate_action = \
+      browser.mainForm.submitSelectWorkflow(value='validate_action',
+                                            maximum_attempt_number=5,
+                                            sleep_between_attempt=5)
+
+  result('Waiting for validate_action', waiting_for_validate_action)
+  result('Show validate', show_validate_time)
+  result('Validated', browser.mainForm.submitDialogConfirm())
+  assert browser.getTransitionMessage() == 'Status changed.'

erp5/util/benchmark/examples/userInfo.py

+# Specify user login/password used to run the tests
+user_tuple = (('zope', 'zope'),)

erp5/util/benchmark/performance_tester.py

+#!/usr/bin/env python
+
+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import argparse
+import os
+import sys
+import multiprocessing
+import xmlrpclib
+import signal
+import errno
+
+from .argument import ArgumentType
+from .process import BenchmarkProcess
+from .result import ERP5BenchmarkResult, CSVBenchmarkResult
+
+MAXIMUM_KEYBOARD_INTERRUPT = 5
+
+class PerformanceTester(object):
+  def __init__(self, namespace=None):
+    if not namespace:
+      self._argument_namespace = self._parse_arguments(argparse.ArgumentParser(
+          description='Run ERP5 benchmarking suites.'))
+    else:
+      self._argument_namespace = namespace
+
+  @staticmethod
+  def _add_parser_arguments(parser):
+    # Optional arguments
+    parser.add_argument('--filename-prefix',
+                        default='result',
+                        metavar='PREFIX',
+                        help='Filename prefix for results and logs files '
+                             '(default: result)')
+
+    parser.add_argument('--report-directory', '-r',
+                        type=ArgumentType.directoryType,
+                        default=os.getcwd(),
+                        metavar='DIRECTORY',
+                        help='Directory where the results and logs will be stored '
+                             '(default: current directory)')
+
+    parser.add_argument('--max-global-average',
+                        type=float,
+                        default=0,
+                        metavar='N',
+                        help='Stop when any suite operation is over this value '
+                             '(default: disable)')
+
+    parser.add_argument('--users-file',
+                        dest='user_info_filename',
+                        default='userInfo',
+                        metavar='MODULE',
+                        help="Import users from ``user_tuple'' in MODULE")
+
+    parser.add_argument('--users-range-increment',
+                        type=ArgumentType.strictlyPositiveIntType,
+                        default=1,
+                        metavar='N',
+                        help='Number of users being added after each repetition '
+                             '(default: 1)')
+
+    parser.add_argument('--enable-debug', '-d',
+                        action='store_true',
+                        default=False,
+                        help='Enable debug messages')
+
+    parser.add_argument('--enable-legacy-listbox',
+                        dest='is_legacy_listbox',
+                        action='store_true',
+                        default=False,
+                        help='Enable legacy listbox for Browser')
+
+    parser.add_argument('--repeat',
+                        type=ArgumentType.strictlyPositiveIntType,
+                        default=-1,
+                        metavar='N',
+                        help='Repeat the benchmark suite N times '
+                             '(default: infinite)')
+
+    parser.add_argument('--user-index',
+                        type=int,
+                        default=0,
+                        metavar='INDEX',
+                        help='Index of the first user within userInfo '
+                             '(default: 0)')
+
+    parser.add_argument('--erp5-publish-url',
+                        metavar='ERP5_PUBLISH_URL',
+                        help='ERP5 URL to publish the results to '
+                             '(default: disabled, thus writing to CSV files)')
+
+    parser.add_argument('--erp5-publish-project',
+                        metavar='ERP5_PUBLISH_PROJECT',
+                        help='ERP5 publish project')
+
+    # Mandatory arguments
+    parser.add_argument('url',
+                        type=ArgumentType.ERP5UrlType,
+                        metavar='URL',
+                        help='ERP5 base URL')
+
+    parser.add_argument('users',
+                        type=ArgumentType.strictlyPositiveIntOrRangeType,
+                        metavar='NB_USERS|MIN_NB_USERS,MAX_NB_USERS',
+                        help='Number of users (fixed or a range)')
+
+    parser.add_argument('benchmark_suite_list',
+                        nargs='+',
+                        metavar='BENCHMARK_SUITES',
+                        help='Benchmark suite modules')
+
+  @staticmethod
+  def _check_parsed_arguments(namespace):
+    namespace.user_tuple = ArgumentType.objectFromModule(namespace.user_info_filename,
+                                                         object_name='user_tuple')
+
+    object_benchmark_suite_list = []
+    for benchmark_suite in namespace.benchmark_suite_list:
+      object_benchmark_suite_list.append(ArgumentType.objectFromModule(benchmark_suite,
+                                                                       callable_object=True))
+
+    namespace.benchmark_suite_name_list = namespace.benchmark_suite_list
+    namespace.benchmark_suite_list = object_benchmark_suite_list
+
+    max_nb_users = isinstance(namespace.users, tuple) and namespace.users[1] or \
+        namespace.users
+
+    namespace.user_tuple = namespace.user_tuple[namespace.user_index:]
+    if max_nb_users > len(namespace.user_tuple):
+      raise argparse.ArgumentTypeError("Not enough users in the given file")
+
+    if (namespace.erp5_publish_url and not namespace.erp5_publish_project) or \
+       (not namespace.erp5_publish_url and namespace.erp5_publish_project):
+      raise argparse.ArgumentTypeError("Publish ERP5 URL and project must "
+                                       "be specified")
+
+    return namespace
+
+  @staticmethod
+  def _parse_arguments(parser):
+    PerformanceTester._add_parser_arguments(parser)
+    namespace = parser.parse_args()
+    PerformanceTester._check_parsed_arguments(namespace)
+    return namespace
+
+  def getResultClass(self):
+    if self._argument_namespace.erp5_publish_url:
+      return ERP5BenchmarkResult
+    else:
+      return CSVBenchmarkResult
+
+  def preRun(self):
+    if not self._argument_namespace.erp5_publish_url:
+      return
+
+    self._argument_namespace.erp5_publish_url += \
+        ERP5BenchmarkResult.createResultDocument(self._argument_namespace.erp5_publish_url,
+                                                 self._argument_namespace.erp5_publish_project,
+                                                 self._argument_namespace.repeat,
+                                                 self._argument_namespace.users)          
+
+  def postRun(self, error_message_set):
+    if not self._argument_namespace.erp5_publish_url:
+      return
+
+    ERP5BenchmarkResult.closeResultDocument(self._argument_namespace.erp5_publish_url,
+                                            error_message_set)
+
+  def _run_constant(self, nb_users):
+    process_list = []
+    exit_msg_queue = multiprocessing.Queue(nb_users)
+
+    result_class = self.getResultClass()
+
+    for user_index in range(nb_users):
+      process = BenchmarkProcess(exit_msg_queue, result_class,
+                                 self._argument_namespace, nb_users,
+                                 user_index)
+
+      process_list.append(process)
+
+    for process in process_list:
+      process.start()
+
+    error_message_set = set()
+    process_terminated_counter = 0
+
+    # Ensure that SIGTERM signal (sent by terminate()) is not sent twice
+    do_exit = False
+
+    while process_terminated_counter != len(process_list):
+      try:
+        error_message = exit_msg_queue.get()
+
+      except KeyboardInterrupt, e:
+        print >>sys.stderr, "\nInterrupted by user, stopping gracefully..."
+        do_exit = True
+
+      # An IOError may be raised  when receiving a SIGINT which interrupts the
+      # blocking system call above and the system call should not be restarted
+      # (using siginterrupt), otherwise the  process will stall forever as its
+      # child has already exited
+      except IOError, e:
+        if e.errno == errno.EINTR:
+          continue
+
+      else:
+        if error_message is not None:
+          error_message_set.add(error_message)
+          do_exit = True
+
+        process_terminated_counter += 1
+
+      # In case of  error or SIGINT, kill the other  children because they are
+      # likely failing as well (especially  because a process only exits after
+      # encountering 10 errors)
+      if do_exit:
+        for process in process_list:
+          if process.is_alive():
+            process.terminate()
+            process.join()
+
+    if error_message_set:
+      return (error_message_set, 1)
+
+    return ((), 0)
+
+  def run(self):
+    error_message_set, exit_status = (), 0
+    self.preRun()
+
+    if isinstance(self._argument_namespace.users, tuple):
+      nb_users, max_users = self._argument_namespace.users
+      while True:
+        error_message_set, exit_status = self._run_constant(nb_users)
+        if exit_status != 0 or nb_users == max_users:
+          break
+
+        nb_users = min(nb_users + self._argument_namespace.users_range_increment,
+                       max_users)
+    else:
+      error_message_set, exit_status = self._run_constant(
+        self._argument_namespace.users)
+
+    self.postRun(error_message_set)
+    return error_message_set, exit_status
+
+def main():
+  error_message_set, exit_status = PerformanceTester().run()
+  for error_message in error_message_set:
+    print >>sys.stderr, "ERROR: %s" % error_message
+
+  sys.exit(exit_status)
+
+if __name__ == '__main__':
+  main()

erp5/util/benchmark/process.py

+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import multiprocessing
+import csv
+import traceback
+import os
+import logging
+import signal
+import sys
+
+from ..test_browser.browser import Browser
+
+MAXIMUM_ERROR_COUNTER = 10
+RESULT_NUMBER_BEFORE_FLUSHING = 100
+
+class BenchmarkProcess(multiprocessing.Process):
+  def __init__(self, exit_msg_queue, result_klass, argument_namespace,
+               nb_users, user_index, *args, **kwargs):
+    self._exit_msg_queue = exit_msg_queue
+    self._result_klass = result_klass
+    self._argument_namespace = argument_namespace
+    self._nb_users = nb_users
+    self._user_index = user_index
+
+    # Initialized when running the test
+    self._browser = None
+    self._current_repeat = 1
+
+    # TODO: Per target error counter instead of global one?
+    self._error_counter = 0
+
+    super(BenchmarkProcess, self).__init__(*args, **kwargs)
+
+  def stopGracefully(self, *args, **kwargs):
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+    raise StopIteration("Interrupted by user or because of an error from "
+                        "another process, flushing remaining results...")
+
+  def getBrowser(self, log_file):
+    info_list = tuple(self._argument_namespace.url) + \
+        tuple(self._argument_namespace.user_tuple[self._user_index])
+
+    return Browser(*info_list,
+                   is_debug=self._argument_namespace.enable_debug,
+                   log_file=log_file,
+                   is_legacy_listbox=self._argument_namespace.is_legacy_listbox)
+
+  def runBenchmarkSuiteList(self, result):
+    for target_idx, target in enumerate(self._argument_namespace.benchmark_suite_list):
+      self._logger.debug("EXECUTE: %s" % target)
+      result.enterSuite(target.__name__)
+
+      try:
+        self._browser.open()
+        target(result, self._browser)
+      except StopIteration:
+        raise
+      except Exception, e:
+        msg = "%s: %s" % (target, traceback.format_exc())
+
+        try:
+          msg += "Last response headers:\n%s\nLast response contents:\n%s" % \
+              (self._browser.headers, self._browser.contents)
+        except:
+          pass
+
+        if (self._current_repeat == 1 or
+            self._error_counter == MAXIMUM_ERROR_COUNTER):
+          raise RuntimeError(msg)
+
+        self._error_counter += 1
+        self._logger.warning(msg)
+
+      for stat in result.getCurrentSuiteStatList():
+        mean = stat.mean
+
+        self._logger.info("%s: min=%.3f, mean=%.3f (+/- %.3f), max=%.3f" % \
+                            (stat.full_label,
+                             stat.minimum,
+                             mean,
+                             stat.standard_deviation,
+                             stat.maximum))
+
+        if (self._argument_namespace.max_global_average and
+            mean > self._argument_namespace.max_global_average):
+          raise RuntimeError("Stopping as mean is greater than maximum "
+                             "global average")
+
+      result.exitSuite()
+
+    result.iterationFinished()
+
+  def run(self):
+    result_instance = self._result_klass(self._argument_namespace,
+                                         self._nb_users,
+                                         self._user_index)
+
+    self._logger = result_instance.getLogger()
+
+    # Ensure the data are flushed before exiting, handled by Result class 
+    # __exit__ block
+    signal.signal(signal.SIGTERM, self.stopGracefully)
+ 
+    # Ignore KeyboardInterrupt as it is handled by the parent process
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+    exit_status = 0
+    exit_msg = None
+
+    try:
+      with result_instance as result:
+        self._browser = self.getBrowser(result_instance.log_file)
+
+        while self._current_repeat != (self._argument_namespace.repeat + 1):
+          self._logger.info("Iteration: %d" % self._current_repeat)
+          self.runBenchmarkSuiteList(result)
+          self._current_repeat += 1
+
+          if not self._current_repeat % RESULT_NUMBER_BEFORE_FLUSHING:
+            result.flush()
+
+    except StopIteration, e:
+      self._logger.error(e)
+
+    except BaseException, e:
+      exit_msg = str(e)
+      exit_status = 1
+
+    self._exit_msg_queue.put(exit_msg)
+    sys.exit(exit_status)

erp5/util/benchmark/report.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# First version: ERP5Mechanize from Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import argparse
+
+def parseArguments():
+  parser = argparse.ArgumentParser(
+    description='Generate reports for ERP5 benchmarking suites.')
+
+  parser.add_argument('--enable-debug',
+                      dest='is_debug',
+                      action='store_true',
+                      default=False,
+                      help='Enable debug messages')
+
+  parser.add_argument('--filename-prefix',
+                      default='result',
+                      metavar='PREFIX',
+                      help='Filename prefix for results CSV files '
+                           '(default: result)')
+
+  parser.add_argument('--output-filename',
+                      default='results.pdf',
+                      metavar='FILENAME',
+                      help='PDF output file (default: results.pdf)')
+
+  parser.add_argument('report_directory',
+                      help='Reports directory')
+
+  namespace = parser.parse_args()
+
+  return namespace
+
+import csv
+
+from .result import BenchmarkResultStatistic
+
+def computeStatisticFromFilenameList(argument_namespace, filename_list):
+  reader_list = []
+  stat_list = []
+  label_list = []
+
+  for filename in filename_list:
+    reader = csv.reader(open(filename, 'rb'), delimiter=',',
+                        quoting=csv.QUOTE_MINIMAL)
+
+    reader_list.append(reader)
+
+    # Get headers
+    row_list = reader.next()
+    if not label_list:
+      label_list = row_list
+      for label in label_list:
+        stat_list.append(BenchmarkResultStatistic(*label.split(': ', 1)))
+
+    if row_list != label_list:
+      raise AssertionError, "ERROR: Result labels: %s != %s" % \
+          (label_list, row_list)
+
+    for row_list in reader:
+      for idx, row in enumerate(row_list):
+        stat_list[idx].add(float(row))
+
+  return stat_list
+
+def formatFloatList(value_list):
+  return [ format(value, ".3f") for value in value_list ]
+
+import numpy
+import pylab
+
+from matplotlib import pyplot, ticker
+
+def drawBarDiagram(pdf, title, stat_list):
+  mean_list = []
+  yerr_list = []
+  minimum_list = []
+  maximum_list = []
+  label_list = []
+  error_list = []
+
+  for stat in stat_list:
+    mean_list.append(stat.mean)
+    yerr_list.append(stat.standard_deviation)
+    minimum_list.append(stat.minimum)
+    maximum_list.append(stat.maximum)
+    label_list.append(stat.label)
+    error_list.append(stat.error_sum)
+
+  min_array = numpy.array(minimum_list)
+  mean_array = numpy.array(mean_list)
+  max_array = numpy.array(maximum_list)
+
+  yerr_lower = numpy.minimum(mean_array - min_array, yerr_list)
+  yerr_upper = numpy.minimum(max_array - mean_array, yerr_list)
+
+  ## Draw diagrams
+  # Create the figure
+  figure = pyplot.figure(figsize=(11.69, 8.29))
+  figure.subplots_adjust(bottom=0.13, right=0.98, top=0.95)
+  pyplot.title(title)
+
+  # Create the axes along with their labels
+  axes = figure.add_subplot(111)
+  axes.set_ylabel('Seconds')
+  axes.set_xticks([])
+
+  axes.yaxis.set_major_locator(ticker.MultipleLocator(0.5))
+  axes.yaxis.set_minor_locator(ticker.MultipleLocator(0.25))
+  axes.yaxis.grid(True, 'major', linewidth=1.5)
+  axes.yaxis.grid(True, 'minor')
+ 
+  # Create the bars
+  ind = numpy.arange(len(label_list))
+  width = 0.33
+
+  min_rects = axes.bar(ind, minimum_list, width, color='y', label='Minimum')
+
+  avg_rects = axes.bar(ind + width, mean_list, width, color='r', label='Mean')
+
+  axes.errorbar(numpy.arange(0.5, len(stat_list)), mean_list,
+                yerr=[yerr_lower, yerr_upper], fmt=None,
+                label='Standard deviation')
+
+  max_rects = axes.bar(ind + width * 2, maximum_list, width, label='Maximum',
+                       color='g')
+
+  # Add the legend of bars
+  axes.legend(loc=0)
+
+  axes.table(rowLabels=['Minimum', 'Average', 'Std. deviation', 'Maximum', 'Errors'],
+             colLabels=label_list,
+             cellText=[formatFloatList(minimum_list),
+                       formatFloatList(mean_list),
+                       formatFloatList(yerr_list),
+                       formatFloatList(maximum_list),
+                       error_list],
+             rowColours=('y', 'r', 'b', 'g', 'w'),
+             loc='bottom',
+             colLoc='center',
+             rowLoc='center',
+             cellLoc='center')
+
+  pdf.savefig()
+  pylab.close()
+
+def drawConcurrentUsersPlot(pdf, title, nb_users_list, stat_list):
+  figure = pyplot.figure(figsize=(11.69, 8.29), frameon=False)
+  figure.subplots_adjust(bottom=0.1, right=0.98, left=0.07, top=0.95)
+  pyplot.title(title)
+  pyplot.grid(True, linewidth=1.5)
+
+  axes = figure.add_subplot(111)
+
+  min_array = numpy.array([stat.minimum for stat in stat_list])
+  mean_array = numpy.array([stat.mean for stat in stat_list])
+  max_array = numpy.array([stat.maximum for stat in stat_list])
+
+  yerr_list = [stat.standard_deviation for stat in stat_list]
+  yerr_lower = numpy.minimum(mean_array - min_array, yerr_list)
+  yerr_upper = numpy.minimum(max_array - mean_array, yerr_list)
+
+  axes.plot(nb_users_list, min_array, 'yo-', label='Minimum')
+
+  axes.errorbar(nb_users_list,
+                mean_array,
+                yerr=[yerr_lower, yerr_upper],
+                color='r',
+                ecolor='b',
+                label='Mean',
+                elinewidth=2,
+                fmt='D-',
+                capsize=10.0)
+
+  axes.plot(nb_users_list, max_array, 'gs-', label='Maximum')
+
+  axes.yaxis.set_major_locator(ticker.MultipleLocator(0.5))
+  axes.yaxis.set_minor_locator(ticker.MultipleLocator(0.25))
+  axes.yaxis.grid(True, 'minor')
+
+  axes.xaxis.set_major_locator(ticker.FixedLocator(nb_users_list))
+
+  axes.set_xticks(nb_users_list)
+  axes.legend(loc=0)
+  axes.set_xlabel('Concurrent users')
+  axes.set_ylabel('Seconds')
+
+  pyplot.xlim(xmin=nb_users_list[0])
+  pdf.savefig()
+  pylab.close()
+
+from matplotlib.backends.backend_pdf import PdfPages
+
+import glob
+import os
+import re
+
+user_re = re.compile('-(\d+)users-')
+
+def generateReport():
+  argument_namespace = parseArguments()
+
+  filename_iter = glob.iglob("%s-*repeat*-*users*-*process*.csv" % os.path.join(
+      argument_namespace.report_directory,
+      argument_namespace.filename_prefix))
+
+  per_nb_users_report_dict = {}
+  for filename in filename_iter:
+    report_dict = per_nb_users_report_dict.setdefault(
+      int(user_re.search(filename).group(1)), {'filename': []})
+
+    report_dict['filename'].append(filename)
+
+  pdf = PdfPages(argument_namespace.output_filename)
+
+  for nb_users, report_dict in per_nb_users_report_dict.items():
+    stat_list = computeStatisticFromFilenameList(
+      argument_namespace, report_dict['filename'])
+
+    title = "Ran suites with %d users" % len(report_dict['filename'])
+    for slice_start_idx in range(0, len(stat_list), 12):
+      if slice_start_idx != 0:
+        title += ' (Ctd.)'
+
+      drawBarDiagram(pdf, title, stat_list[slice_start_idx:slice_start_idx + 12])
+
+    report_dict['stats'] = stat_list
+
+  if len(per_nb_users_report_dict) != 1:
+    for i in range(len(report_dict['stats'])):
+      stat_list = []
+      nb_users_list = per_nb_users_report_dict.keys()
+      for report_dict in per_nb_users_report_dict.values():
+        stat_list.append(report_dict['stats'][i])
+
+      drawConcurrentUsersPlot(
+        pdf,
+        "%s from %d to %d users (step: %d)" % (stat_list[0].full_label,
+                                               nb_users_list[0],
+                                               nb_users_list[-1],
+                                               nb_users_list[1] - nb_users_list[0]),
+        nb_users_list,
+        stat_list)
+
+  pdf.close()
+
+if __name__ == '__main__':
+  generateReport()

erp5/util/benchmark/result.py

+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import sys
+import math
+import os
+import csv
+import logging
+import signal
+
+class BenchmarkResultStatistic(object):
+  def __init__(self, suite, label):
+    self.suite = suite
+    self.label = label
+
+    self.full_label = '%s: %s' % (self.suite, self.label)
+
+    self.minimum = sys.maxint
+    self.maximum = -1
+    self.n = 0
+    self.error_sum = 0
+
+    # For calculating the mean
+    self._value_sum = 0
+
+    # For calculating the standard deviation
+    self._variance_sum = 0
+    self._mean = 0
+
+  def add_error(self):
+    self.error_sum += 1
+
+  def add(self, value):
+    if value < self.minimum:
+      self.minimum = value
+    if value > self.maximum:
+      self.maximum = value
+
+    self._value_sum += value
+    self.n += 1
+
+    delta = value - self._mean
+    self._mean += delta / self.n
+    self._variance_sum += delta * (value - self._mean)
+
+  @property
+  def mean(self):
+    return self._value_sum / self.n
+
+  @property
+  def standard_deviation(self):
+    return math.sqrt(self._variance_sum / self.n)
+
+import abc
+
+class BenchmarkResult(object):
+  __metaclass__ = abc.ABCMeta
+
+  def __init__(self, argument_namespace, nb_users, user_index):
+    self._argument_namespace = argument_namespace
+    self._nb_users = nb_users
+    self._user_index = user_index
+
+    self._log_level = self._argument_namespace.enable_debug and \
+        logging.DEBUG or logging.INFO
+
+    self._stat_list = []
+    self._suite_idx = 0
+    self._result_idx = 0
+    self.result_list = []
+    self._all_result_list = []
+    self._first_iteration = True
+    self._current_suite_name = None
+    self._result_idx_checkpoint_list = []
+    self.label_list = []
+    self._logger = None
+
+  def getLogger(self):
+    if not self._logger:
+      logging.basicConfig(stream=self.log_file, level=self._log_level)
+      self._logger = logging.getLogger('erp5.util.benchmark')
+      return self._logger
+
+    return self._logger
+
+  def __enter__(self):
+    return self
+
+  def enterSuite(self, name):
+    self._current_suite_name = name
+
+  def __call__(self, label, value):
+    self.result_list.append(value)
+    if self._first_iteration:
+      self._stat_list.append(BenchmarkResultStatistic(self._current_suite_name,
+                                                      label))
+
+    self._stat_list[self._result_idx].add(value)
+    self._result_idx += 1
+
+  def getLabelList(self):
+    return [ stat.full_label for stat in self._stat_list ]
+
+  def iterationFinished(self):
+    self._all_result_list.append(self.result_list)
+    if self._first_iteration:
+      self.label_list = self.getLabelList()
+
+    self.getLogger().debug("RESULTS: %s" % self.result_list)
+    self.result_list = []
+    self._first_iteration = False
+    self._suite_idx = 0
+    self._result_idx = 0
+
+  def getStatList(self):
+    return self._stat_list
+
+  def getCurrentSuiteStatList(self):
+    start_index = self._suite_idx and \
+        self._result_idx_checkpoint_list[self._suite_idx - 1] or 0
+
+    return self._stat_list[start_index:self._result_idx]
+
+  def exitSuite(self):
+    if self._first_iteration:
+      self._result_idx_checkpoint_list.append(self._result_idx)
+    else:
+      expected_result_idx = self._result_idx_checkpoint_list[self._suite_idx]
+      while self._result_idx != expected_result_idx:
+        self.result_list.append(0)
+        self._stat_list[self._result_idx].add_error()
+        self._result_idx += 1
+
+    self._suite_idx += 1
+
+  @abc.abstractmethod
+  def flush(self, partial=True):
+    self._all_result_list = []
+
+  @abc.abstractmethod
+  def __exit__(self, exc_type, exc_value, traceback):
+    signal.signal(signal.SIGTERM, signal.SIG_IGN)
+    self.flush(partial=False)
+    return True
+
+class CSVBenchmarkResult(BenchmarkResult):
+  def __init__(self, *args, **kwargs):
+    super(CSVBenchmarkResult, self).__init__(*args, **kwargs)
+
+    filename_prefix = self._getFilenamePrefix()
+
+    self._result_filename = "%s.csv" % filename_prefix
+    self._result_filename_path = os.path.join(
+      self._argument_namespace.report_directory, self._result_filename)
+
+    self._log_filename = "%s.log" % filename_prefix
+    self._log_filename_path = os.path.join(
+      self._argument_namespace.report_directory, self._log_filename)
+
+    self.log_file = open(self._log_filename_path, 'w')
+
+  def _getFilenamePrefix(self):
+    max_nb_users = isinstance(self._argument_namespace.users, int) and \
+        self._argument_namespace.users or self._argument_namespace.users[1]
+
+    fmt = "%%s-%%drepeat-%%0%ddusers-process%%0%dd" % \
+        (len(str(max_nb_users)), len(str(self._nb_users)))
+
+    return fmt % (self._argument_namespace.filename_prefix,
+                  self._argument_namespace.repeat,
+                  self._nb_users,
+                  self._user_index)
+
+  def __enter__(self):
+    self._result_file = open(self._result_filename_path, 'wb')
+    self._csv_writer = csv.writer(self._result_file, delimiter=',',
+                                  quoting=csv.QUOTE_MINIMAL)
+
+    return self
+
+  def flush(self, partial=True):
+    if self._result_file.tell() == 0:
+      self._csv_writer.writerow(self.label_list)
+
+    self._csv_writer.writerows(self._all_result_list)
+    self._result_file.flush()
+    os.fsync(self._result_file.fileno())
+
+    super(CSVBenchmarkResult, self).flush(partial)
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    super(CSVBenchmarkResult, self).__exit__(exc_type, exc_value, traceback)
+    self._result_file.close()
+
+    if exc_type and not issubclass(exc_type, StopIteration):
+      msg = "An error occured, see: %s" % self._log_filename_path
+      self.getLogger().error("%s: %s" % (exc_type, exc_value))
+      raise RuntimeError(msg)
+
+from cStringIO import StringIO
+
+import xmlrpclib
+import datetime
+
+class ERP5BenchmarkResult(BenchmarkResult):
+  def __init__(self, *args, **kwargs):
+    self.log_file = StringIO()
+    self._log_buffer_list = []
+
+    super(ERP5BenchmarkResult, self).__init__(*args, **kwargs)
+
+  def iterationFinished(self):
+    super(ERP5BenchmarkResult, self).iterationFinished()
+
+    # TODO: garbage?
+    self._log_buffer_list.append(self.log_file.getvalue())
+    self.log_file.seek(0)
+
+  def flush(self, partial=True):
+    benchmark_result = xmlrpclib.ServerProxy(
+      self._argument_namespace.erp5_publish_url,
+      verbose=True,
+      allow_none=True)
+
+    benchmark_result.BenchmarkResult_addResultLineList(
+      self._argument_namespace.user_tuple[self._user_index][0],
+      self._argument_namespace.repeat,
+      self._nb_users,
+      self._argument_namespace.benchmark_suite_name_list,
+      self.getLabelList(),
+      self._all_result_list,
+      self._log_buffer_list)
+
+    super(ERP5BenchmarkResult, self).flush()
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    super(ERP5BenchmarkResult, self).__exit__(exc_type, exc_value, traceback)
+
+  @staticmethod
+  def createResultDocument(publish_url, publish_project, repeat, nb_users):
+    test_result_module = xmlrpclib.ServerProxy(publish_url,
+                                               verbose=True,
+                                               allow_none=True)
+
+    if isinstance(nb_users, tuple):
+      nb_users_str = '%d to %d' % nb_users
+    else:
+      nb_users_str = '%d' % nb_users
+
+    benchmark_result = test_result_module.TestResultModule_addBenchmarkResult(
+      '%d repeat with %s concurrent users' % (repeat, nb_users_str),
+      publish_project, ' '.join(sys.argv), datetime.datetime.now())
+
+    return benchmark_result['id']
+
+  @staticmethod
+  def closeResultDocument(publish_document_url, error_message_set):
+    result = xmlrpclib.ServerProxy(publish_document_url,
+                                   verbose=True,
+                                   allow_none=True)
+
+    result.BenchmarkResult_completed(error_message_set and 'FAIL' or 'PASS',
+                                     error_message_set)

erp5/util/benchmark/scalability_tester.py

+#!/usr/bin/env python
+
+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from .result import CSVBenchmarkResult
+
+class CSVScalabilityBenchmarkResult(CSVBenchmarkResult):
+  def flush(self, partial=True):
+    super(CSVScalabilityBenchmarkResult, self).flush(partial)
+    self._argument_namespace.notify_method(self._result_filename,
+                                           self._result_file.tell(),
+                                           partial=partial)
+
+from .performance_tester import PerformanceTester
+
+class ScalabilityTester(PerformanceTester):
+  def preRun(self, *args, **kwargs):
+    pass
+
+  def postRun(self, error_message_set):
+    from logging import Formatter
+    import sys
+    import urllib
+    import urllib2
+
+    try:
+      urllib2.urlopen("http://[%s]:%d/report" % \
+                        (self._argument_namespace.manager_address,
+                         self._argument_namespace.manager_port),
+                      urllib.urlencode({'error_message_set': '|'.join(error_message_set)})).close()
+
+    except:
+      print >>sys.stderr, "ERROR: %s" % Formatter().formatException(sys.exc_info())
+
+  def getResultClass(self):
+    if not self._argument_namespace.erp5_publish_url:
+      return CSVScalabilityBenchmarkResult
+
+    return super(ScalabilityTester, self).getResultClass()
+
+from slapos.tool.nosqltester import NoSQLTester
+
+class RunScalabilityTester(NoSQLTester):
+  def __init__(self):
+    super(RunScalabilityTester, self).__init__()
+
+  def _add_parser_arguments(self, parser):
+    super(RunScalabilityTester, self)._add_parser_arguments(parser)
+    ScalabilityTester._add_parser_arguments(parser)
+
+  def _parse_arguments(self, parser):
+    namespace = super(RunScalabilityTester, self)._parse_arguments(parser)
+    ScalabilityTester._check_parsed_arguments(namespace)
+    namespace.notify_method = self.send_result_availability_notification
+    return namespace
+
+  def run_tester(self):
+    ScalabilityTester(self.argument_namespace).run()
+
+def main():
+  RunScalabilityTester().run()
+
+if __name__ == '__main__':
+  main()

erp5/util/test_browser/browser.py

+# -*- coding: utf-8 -*-
+
+##############################################################################
+#
+# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
+#                    Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+#
+# First version: ERP5Mechanize from Vincent Pelletier <vincent@nexedi.com>
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsability of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# garantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import logging
+
+import sys
+from urlparse import urljoin
+from z3c.etestbrowser.browser import ExtendedTestBrowser
+from zope.testbrowser.browser import onlyOne
+
+def measurementMetaClass(prefix):
+  """
+  Prepare a meta class where the C{prefix} is used to select for which
+  methods measurement methods will be added automatically.
+
+  @param prefix:
+  @type prefix: str
+  @return: The measurement meta class corresponding to the prefix
+  @rtype: type
+  """
+  class MeasurementMetaClass(type):
+    """
+    Meta class to automatically wrap methods whose prefix starts with
+    C{prefix}, and also to define C{lastRequestSeconds} on other classes
+    besides of Browser.
+    """
+    def __new__(metacls, name, bases, dictionary):
+      def timeInSecondDecorator(method):
+        def wrapper(self, *args, **kwargs):
+          """
+          Replaced by the wrapped method docstring. Some methods return the
+          time spent on waiting (C{submitSelectJump} and for example), thus
+          return a tuple with the time spent on the request and the time spent
+          on waiting
+          """
+          ret = method(self, *args, **kwargs)
+          if ret is None:
+            return self.lastRequestSeconds
+          else:
+            return (self.lastRequestSeconds, ret)
+
+        return wrapper
+
+      def applyMeasure(method):
+        """
+        Inner function to wrap timed methods to automatically return the time
+        spent on the request
+
+        @param method: Instance method to be called
+        @type method: function
+        """
+        wrapper_method = timeInSecondDecorator(method)
+        wrapper_method.func_name = method.func_name
+        wrapper_method.__doc__ = method.__doc__
+
+        # In order to avoid re-wrapping the method when looking at the bases
+        # for example
+        wrapper_method.__is_wrapper__ = True
+
+        dictionary[method.func_name] = wrapper_method
+
+      # Only wrap methods prefixed by the given prefix
+      for attribute_name, attribute in dictionary.items():
+        if attribute_name.startswith(prefix) and callable(attribute):
+          applyMeasure(attribute)
+
+      # And also create these methods by looking at the bases
+      for attribute_name in dir(bases[0]):
+        if attribute_name not in dictionary and \
+           attribute_name.startswith(prefix):
+          attribute = getattr(bases[0], attribute_name)
+          if callable(attribute) and not getattr(attribute, '__is_wrapper__', False):
+            applyMeasure(attribute)
+
+      # lastRequestSeconds properties are only defined on classes inheriting
+      # from zope.testbrowser.browser.Browser, so create these properties for
+      # all other classes too
+      if 'Browser' not in bases[0].__name__:
+        time_method = lambda self: self.browser.lastRequestSeconds
+        time_method.func_name = 'lastRequestSeconds'
+        time_method.__doc__ = Browser.lastRequestSeconds.__doc__
+        dictionary['lastRequestSeconds'] = property(time_method)
+
+      return super(MeasurementMetaClass,
+                   metacls).__new__(metacls, name, bases, dictionary)
+
+  return MeasurementMetaClass
+
+class Browser(ExtendedTestBrowser):
+  """
+  Implements mechanize tests specific to an ERP5 environment through
+  U{ExtendedTestBrowser<http://pypi.python.org/pypi/z3c.etestbrowser>}
+  (providing features to parse XML and access elements using XPATH)
+  using U{zope.testbrowser<http://pypi.python.org/pypi/zope.testbrowser>}
+  (providing benchmark and testing features on top of
+  U{mechanize<http://wwwsearch.sourceforge.net/mechanize/>}).
+
+  @todo:
+   - getFormulatorFieldValue
+  """
+  __metaclass__ = measurementMetaClass(prefix='open')
+
+  def __init__(self,
+               base_url,
+               erp5_site_id,
+               username,
+               password,
+               log_file=None,
+               is_debug=False,
+               is_legacy_listbox=False):
+    """
+    Create a browser object, allowing to log in right away with the
+    given username and password. The base URL must contain an I{/} at
+    the end.
+
+    @param base_url: Base HTTP URL
+    @type base_url: str
+    @param erp5_site_id: ERP5 site name
+    @type erp5_site_id: str
+    @param username: Username to be used to log into ERP5
+    @type username: str
+    @param password: Password to be used to log into ERP5
+    @param log_file: Log file object (stderr if none given)
+    @type log_file: file
+    @param is_debug: Enable or disable debugging (disable by default)
+    @type is_debug: bool
+    @param is_legacy_listbox: Use legacy listbox
+    @type is_legacy_listbox: bool
+    """
+    # Meaningful to re-create the MainForm class every time the page
+    # has been changed
+    self._main_form = None
+
+    assert base_url[-1] == '/'
+
+    self._base_url = base_url
+    self._erp5_site_id = erp5_site_id
+    self._erp5_base_url = urljoin(self._base_url, self._erp5_site_id) + '/'
+
+    self._username = username
+    self._password = password
+
+    # Only display WARNING message if debugging is not enabled
+    logging_level = is_debug and logging.DEBUG or logging.WARNING
+    stream = log_file and log_file or sys.stderr
+    logging.basicConfig(stream=stream, level=logging_level)
+
+    self._logger = logging.getLogger('erp5.util.test_browser')
+    self._is_legacy_listbox = is_legacy_listbox
+    self._is_logged_in = False
+
+    super(Browser, self).__init__()
+
+    self.login()
+
+  def open(self, url_or_path=None, data=None):
+    """
+    Open a relative (to the ERP5 base URL) or absolute URL. If the
+    given URL is not given, then it will open the home ERP5 page.
+
+    @param url_or_path: Relative or absolute URL
+    @type url_or_path: str
+    """
+    # In case url_or_path is an absolute URL, urljoin() will return
+    # it, otherwise it is a relative path and will be concatenated to
+    # ERP5 base URL
+    absolute_url = urljoin(self._erp5_base_url, url_or_path)
+
+    self._logger.debug("Opening url: " + absolute_url)
+    super(Browser, self).open(absolute_url, data)
+
+  def login(self, force=False):
+    """
+    Log in only if not already logged in unless explicitely specified
+    to do so.
+
+    @param force: Log in even if already logged in
+    @type force: bool
+    """
+    if force or not self._is_logged_in:
+      try:
+        url_before_login = self.url
+      except:
+        url_before_login = None
+
+      self.open('login_form')
+      self.mainForm.submitLogin()
+
+      # Go back to the page before trying to log in if any URL, or to
+      # the homepage otherwise
+      self.open(url_before_login)
+
+  def getCookieValue(self, name, default=None):
+    """
+    Get the cookie value of the given cookie name.
+
+    @param name: Name of the cookie
+    @type name: str
+    @param default: Fallback value if the cookie was not found
+    @type default: str
+    @return: Cookie value
+    @rtype: str
+    """
+    for cookie_name, cookie_value in self.cookies.iteritems():
+      if name == cookie_name:
+        return cookie_value
+
+    return default
+
+  @property
+  def mainForm(self):
+    """
+    Get the ERP5 main form of the current page. ERP5 generally use
+    only one form (whose C{id} is C{main_form}) for all the controls
+    within a page. A Form instance is returned including helper
+    methods specific to ERP5.
+
+    @return: The main Form class instance
+    @rtype: Form
+
+    @raise LookupError: The main form could not be found.
+
+    @todo: Perhaps the page could be parsed to generate a class with
+           only appropriate methods, but that would certainly be an
+           huge overhead for little benefit...
+
+    @todo: Patch zope.testbrowser to allow the class to be given
+           rather than duplicating the code
+    """
+    # If the page has not changed, no need to re-create a class, so
+    # just return the main_form instance
+    if self._main_form and self._counter == self._main_form._browser_counter:
+      return self._main_form
+
+    main_form = None
+    for form in self.mech_browser.forms():
+      if form.attrs.get('id') == 'main_form':
+        main_form = form
+
+    if not main_form:
+      raise LookupError("Could not get 'main_form'")
+
+    self.mech_browser.form = form
+    self._main_form = ContextMainForm(self, form)
+    return self._main_form
+
+  def getLink(self, text=None, url=None, id=None, index=0,
+              class_attribute=None):
+    """
+    Override original C{getLink} allowing to not consider the HTTP
+    query string unless it is explicitly given.
+
+    Also, allows to select a link by its class attribute, which
+    basically look for the first element whose C{attribute} is
+    C{class_attribute} then call C{getLink} with the element C{href}.
+
+    @param class_attribute: Get the link with this class
+    @type class_attribute: str
+    @param args: Positional arguments given to original C{getLink}
+    @type args: list
+    @param kwargs: Keyword arguments given to original C{getLink}
+    @type kwargs: dict
+    """
+    if class_attribute:
+      element_list = self.etree.xpath('//a[contains(@class, "%s")]' % \
+                                        class_attribute)
+
+      try:
+        url = element_list[0].get('href')
+      except (IndexError, AttributeError):
+        url = None
+
+      if not url:
+        raise LookupError("Could not find any link whose class is '%s'" % \
+                            class_attribute)
+
+    elif url and '?' not in url:
+      url += '?'
+
+    if id is not None:
+      def predicate(link):
+        return dict(link.attrs).get('id') == id
+      args = {'predicate': predicate}
+    else:
+      import re
+      from zope.testbrowser.browser import RegexType
+
+      if isinstance(text, RegexType):
+        text_regex = text
+      elif text is not None:
+        text_regex = re.compile(re.escape(text), re.DOTALL)
+      else:
+        text_regex = None
+
+      if isinstance(url, RegexType):
+        url_regex = url
+      elif url is not None:
+        url_regex = re.compile(re.escape(url), re.DOTALL)
+      else:
+        url_regex = None
+      args = {'text_regex': text_regex, 'url_regex': url_regex}
+
+    args['nr'] = index
+    return LinkWithTime(self.mech_browser.find_link(**args), self)
+
+  def getImportExportLink(self):
+    """
+    Get Import/Export link. Use the class attribute rather than the
+    name as the latter is dependent on the context.
+
+    @return: The link whose class is C{report}
+    @rtype: Link
+
+    @todo: Should perhaps be a ContextBrowser class?
+    """
+    return self.getLink(class_attribute='import_export')
+
+  def getFastInputLink(self):
+    """
+    Get Fast Input link. Use the class attribute rather than the name
+    as the latter is dependent on the context.
+
+    @return: The link whose class is C{fast_input}
+    @rtype: Link
+
+    @todo: Should perhaps be a ContextBrowser class?
+    """
+    return self.getLink(class_attribute='fast_input')
+
+  def getTransitionMessage(self):
+    """
+    Parse the current page and returns the value of the portal_status
+    message.
+
+    @return: The transition message
+    @rtype: str
+
+    @raise LookupError: Not found
+    """
+    try:
+      return self.etree.xpath('//div[@id="transition_message"]')[0].text
+    except IndexError:
+      raise LookupError("Cannot find div with ID 'transition_message'")
+
+  def getInformationArea(self):
+    """
+    Parse the current page and returns the value of the information_area
+    message.
+
+    @return: The information area message
+    @rtype: str
+
+    @raise LookupError: Not found
+    """
+    try:
+      return self.etree.xpath('//div[@id="information_area"]')[0].text
+    except IndexError:
+      raise LookupError("Cannot find div with ID 'information_area'")
+
+  _listbox_table_xpath_str = '//table[contains(@class, "listbox-table")]'
+
+  _legacy_listbox_table_xpath_str = '//div[contains(@class, "listbox")]'\
+      '//table'
+
+  def getListboxLink(self, line_number, column_number, cell_element_index=1,
+                     *args, **kwargs):
+    """
+    Follow the link at the given position, excluding any link whose
+    class is hidden. In case there are several links within a cell,
+    C{cell_element_index} allows to select which one to get (starting
+    from 1).
+
+    @param line_number: Line number of the link
+    @type line_number: int
+    @param column_number: Column number of the link
+    @type column_number: int
+    @param cell_element_index: Index of the link to be selected in the cell
+    @type cell_element_index: int
+    @param args: positional arguments given to C{getLink}
+    @type args: list
+    @param kwargs: keyword arguments given to C{getLink}
+    @type kwargs: dict
+    @return: C{Link} at the given line and column number
+    @rtype: L{zope.testbrowser.interfaces.ILink}
+
+    @raise LookupError: No link could be found at the given position
+                        and cell indexes
+    """
+    if self._is_legacy_listbox:
+      listbox_basic_xpath_str = self._legacy_listbox_table_xpath_str
+    else:
+      listbox_basic_xpath_str = self._listbox_table_xpath_str
+
+    # With XPATH, the position is context-dependent, therefore, as
+    # there the cells are either within a <thead> or <tbody>, the line
+    # number must be shifted by the number of header lines (namely 2)
+    if line_number <= 2:
+      relative_line_number = line_number
+
+      if self._is_legacy_listbox:
+        column_type = 'td'
+      else:
+        column_type = 'th'
+    else:
+      if self._is_legacy_listbox:
+        relative_line_number = line_number
+      else:
+        relative_line_number = line_number - 2
+
+      column_type = 'td'
+
+    xpath_str = '%s//tr[%d]//%s[%d]//a[not(contains(@class, "hidden"))][%d]' % \
+        (listbox_basic_xpath_str,
+         relative_line_number,
+         column_type,
+         column_number,
+         cell_element_index)
+
+    # xpath() method always return a list even if there is only one element
+    element_list = self.etree.xpath(xpath_str)
+
+    try:
+      link_href = element_list[0].get('href')
+    except (IndexError, AttributeError):
+      link_href = None
+
+    if not link_href:
+      raise LookupError("Could not find link in listbox cell %dx%d (index=%d)" %\
+                          (line_number, column_number, cell_element_index))
+
+    return self.getLink(url=link_href, *args, **kwargs)
+
+  def getListboxPosition(self,
+                         text,
+                         column_number=None,
+                         line_number=None,
+                         strict=False):
+    """
+    Returns the position number of the first line containing given
+    text in given column or line number (starting from 1).
+
+    @param text: Text to search
+    @type text: str
+    @param column_number: Look into all the cells of this column
+    @type column_number: int
+    @param line_number: Look into all the cells of this line
+    @type line_number: int
+    @param strict: Should given text matches exactly
+    @type strict: bool
+    @return: The cell position
+    @rtype: int
+
+    @raise LookupError: Not found
+    """
+    # Require either column_number or line_number to be given
+    onlyOne([column_number, line_number], '"column_number" and "line_number"')
+
+    if self._is_legacy_listbox:
+      listbox_basic_xpath_str = self._legacy_listbox_table_xpath_str
+    else:
+      listbox_basic_xpath_str = self._listbox_table_xpath_str
+
+    # Get all cells in the column (if column_number is given and
+    # including header columns) or line (if line_number is given)
+    if column_number:
+      xpath_str_fmt = listbox_basic_xpath_str + '//tr//%%s[%d]' % \
+          column_number
+
+      if self._is_legacy_listbox:
+        column_or_line_xpath_str = xpath_str_fmt % 'td'
+      else:
+        column_or_line_xpath_str = "%s | %s" % (xpath_str_fmt % 'th',
+                                                xpath_str_fmt % 'td')
+
+    else:
+      listbox_basic_xpath_str = self._listbox_table_xpath_str
+
+      # With XPATH, the position is context-dependent, therefore, as
+      # there the cells are either within a <thead> or <tbody>, the
+      # line number must be shifted by the number of header lines
+      # (namely 2)
+      if line_number <= 2:
+        relative_line_number = line_number
+
+        if self._is_legacy_listbox:
+          column_type = 'td'
+        else:
+          column_type = 'th'
+      else:
+        if self._is_legacy_listbox:
+          relative_line_number = line_number
+        else:
+          relative_line_number = line_number - 2
+
+        column_type = 'td'
+
+      column_or_line_xpath_str = listbox_basic_xpath_str + '//tr[%d]//%s' %\
+          (relative_line_number, column_type)
+
+    cell_list = self.etree.xpath(column_or_line_xpath_str)
+
+    # Iterate over the cells list until one the children content
+    # matches the expected text
+    for position, cell in enumerate(cell_list):
+      for child in cell.iterchildren():
+        if not child.text:
+          continue
+
+        if (strict and child.text == text) or \
+           (not strict and text in child.text):
+          return position + 1
+
+    raise LookupError("No matching cell with value '%s'" % text)
+
+  def getRemainingActivityCounter(self):
+    """
+    Return the number of remaining activities, but do not visit the
+    URL so it does not interfere with next calls.
+
+    @return: The number of remaining activities
+    @rtype: int
+    """
+    self._logger.debug("Checking the number of remaining activities")
+    activity_counter = self.mech_browser.open_novisit(
+      self._erp5_base_url + 'portal_activities/countMessage').read()
+
+    activity_counter = activity_counter and int(activity_counter) or 0
+    self._logger.debug("Remaining activities: %d" % activity_counter)
+
+    return activity_counter
+
+from zope.testbrowser.browser import Form, ListControl
+
+class LoginError(Exception):
+  """
+  Exception raised when login fails
+  """
+  pass
+
+class MainForm(Form):
+  """
+  Class defining convenient methods for the main form of ERP5. All the
+  methods specified are those always found in an ERP5 page in contrary
+  to L{ContextMainForm}.
+  """
+  __metaclass__ = measurementMetaClass(prefix='submit')
+
+  def submit(self, label=None, name=None, class_attribute=None, index=None,
+             *args, **kwargs):
+    """
+    Overriden for logging purpose, and for specifying a default index
+    to 0 if not set, thus avoiding AmbiguityError being raised (in
+    ERP5 there may be several submit fields with the same name).
+
+    Also, allows to select a submit by its class attribute, which
+    basically look for the first element whose C{attribute} is
+    C{class_attribute} then call C{submit} with the element C{name}.
+
+    @param class_attribute: Submit according to the class attribute
+    @type class_attribute: str
+
+    @raise LookupError: Could not find any element matching the given
+                        class attribute name, if class_attribute
+                        parameter is given.
+    """
+    self.browser._logger.debug(
+      "Submitting (name='%s', label='%s', class='%s')" % (name, label,
+                                                          class_attribute))
+
+    if class_attribute:
+      element_list = self.browser.etree.xpath('//*[contains(@class, "%s")]' % \
+                                                class_attribute)
+
+      try:
+        name = element_list[0].get('name')
+      except (IndexError, AttributeError):
+        name = None
+
+      if not name:
+        raise LookupError("Could not find any button whose class is '%s'" % \
+                            class_attribute)
+
+    if label is None and name is None:
+      super(MainForm, self).submit(label=label, name=name, *args, **kwargs)
+    else:
+      if index is None:
+        index = 0
+
+      super(MainForm, self).submit(label=label, name=name, index=index,
+                                   *args, **kwargs)
+
+  def submitSelect(self, select_name, submit_name, label=None, value=None,
+                   select_index=None, control_index=None):
+    """
+    Get the select control whose name attribute is C{select_name},
+    then select the option control specified either by its C{label} or
+    C{value} within that select control, and finally submit it using
+    the submit control whose name attribute is C{submit_name}.
+
+    The C{value} matches an option value if found at the end of the
+    latter (excluding the query string), for example a search for
+    I{/logout} will match I{/erp5/logout} and I{/erp5/logout?foo=bar}
+    (if and only if C{value} contains no query string) but not
+    I{/erp5/logout_bar}.
+
+    Label value is searched as case-sensitive whole words within the
+    labels for each item--that is, a search for I{Add} will match
+    I{Add a contact} but not I{Address}.  A word is defined as one or
+    more alphanumeric characters or the underline.
+
+    C{select_index} and C{control_index} have the same meaning as in
+    zope.testbrowser, namely to select a particular select or control
+    when the C{label} or C{value} is ambiguous.
+
+    @param select_name: Select control name
+    @type select_name: str
+    @param submit_name: Submit control name
+    @type submit_name: str
+    @param label: Label of the option control
+    @type label: str
+    @param value: Value of the option control
+    @type value: str
+    @param select_index: Index of the select if multiple matches
+    @type select_index: int
+    @param control_index: Index of the control if multiple matches
+    @type control_index: int
+
+    @raise LookupError: The select, option or submit control could not
+                        be found
+    """
+    select_control = self.getControl(name=select_name, index=select_index)
+
+    # zope.testbrowser checks for a whole word but it is also useful
+    # to match the end of the option control value string because in
+    # ERP5, the value could be URL (such as 'http://foo:81/erp5/logout')
+    if value:
+      for item in select_control.options:
+        if '?' not in value:
+          item = item.split('?')[0]
+
+        if item.endswith(value):
+          value = item
+          break
+
+    self.browser._logger.debug("select_id='%s', label='%s', value='%s'" % \
+                                 (select_name, label, value))
+
+    select_control.getControl(label=label, value=value,
+                              index=control_index).selected = True
+
+    self.submit(name=submit_name)
+
+  def submitLogin(self):
+    """
+    Log into ERP5 using the username and password provided in the
+    browser. It is assumed that the current page is the login page (by
+    calling C{open('login_form')} beforehand).
+
+    This method should rarely be used by scripts as login is already
+    performed upon instanciation of Browser class.
+
+    @raise LoginError: Login failed
+
+    @todo: Use information sent back as headers rather than looking
+           into the page content?
+    """
+    self.browser._logger.debug("Logging in: username='%s', password='%s'" % \
+                                 (self.browser._username, self.browser._password))
+
+    self.getControl(name='__ac_name').value = self.browser._username
+    self.getControl(name='__ac_password').value = self.browser._password
+    self.submit()
+
+    if 'Logged In as' not in self.browser.contents:
+      raise LoginError("%s: Could not log in as '%s:%s'" % \
+                         (self.browser._erp5_base_url,
+                          self.browser._username,
+                          self.browser._password))
+
+    self.browser._is_logged_in = True
+
+  def submitSelectFavourite(self, label=None, value=None, **kw):
+    """
+    Select and submit a favourite, given either by its label (such as
+    I{Log out}) or value (I{/logout}). See L{submitSelect}.
+    """
+    self.submitSelect('select_favorite', 'Base_doFavorite:method', label, value,
+                      **kw)
+
+  def submitSelectModule(self, label=None, value=None, **kw):
+    """
+    Select and submit a module, given either by its label (such as
+    I{Currencies}) or value (such as I{/glossary_module}). See
+    L{submitSelect}.
+    """
+    self.submitSelect('select_module', 'Base_doModule:method', label, value,
+                      **kw)
+
+  def submitSelectLanguage(self, label=None, value=None, **kw):
+    """
+    Select and submit a language, given either by its label (such as
+    I{English}) or value (such as I{en}). See L{submitSelect}.
+    """
+    self.submitSelect('select_language', 'Base_doLanguage:method', label, value)
+
+  def submitSearch(self, search_text):
+    """
+    Fill search field with C{search_text} and submit it.
+
+    @param search_text: Text to search
+    @type search_text: str
+    """
+    self.getControl(name='field_your_search_text').value = search_text
+    self.submit(name='ERP5Site_viewQuickSearchResultList:method')
+
+  def submitLogout(self):
+    """
+    Perform logout.
+    """
+    self.submitSelectFavourite(value='/logout')
+
+import time
+
+class ContextMainForm(MainForm):
+  """
+  Class defining context-dependent convenient methods for the main
+  form of ERP5.
+
+  @todo:
+   - doListboxAction
+   - doContextListMode
+   - doContextSearch
+   - doContextSort
+   - doContextConfigure
+   - doContextButton
+   - doContextReport
+   - doContextExchange
+  """
+  def submitSelectJump(self, label=None, value=None,
+                       no_jump_transition_message=None,
+                       maximum_attempt_number=1, sleep_between_attempt=0,
+                       **kw):
+    """
+    Select and submit a jump, given either by its label (such as
+    I{Queries}) or value (such as
+    I{/person_module/Base_jumpToRelatedObject?portal_type=Foo}). See
+    L{submitSelect}.
+
+    Usually, a transition message will be displayed if it was not possible to
+    jump (for example because the object has not been created yet), therefore
+    the number of attempts before failing can be specified if necessary.
+
+    @param no_jump_transition_message: Transition message displayed if the
+                                       jump could not be performed
+    @type no_jump_transition_message: str
+    @param maximum_attempt_number: Number of attempts before failing
+    @type maximum_attempt_number: int
+    @param sleep_between_attempt: Sleep N seconds between attempts
+    @type sleep_between_attempt: int
+    """
+    if not no_jump_transition_message:
+      self.submitSelect('select_jump', 'Base_doJump:method',
+                        label, value, **kw)
+    else:
+      current_attempt_counter = 0
+      while current_attempt_counter != maximum_attempt_number:
+        self.browser.mainForm.submitSelect('select_jump', 'Base_doJump:method',
+                                           label, value, **kw)
+
+        if no_jump_transition_message != self.browser.getTransitionMessage():
+          return current_attempt_counter * sleep_between_attempt
+
+        time.sleep(sleep_between_attempt)
+        current_attempt_counter += 1
+
+      raise AssertionError("Could not jump to related object")
+
+  def submitSelectAction(self, label=None, value=None, **kw):
+    """
+    Select and submit an action, given either by its label (such as
+    I{Add Person}) or value (such as I{add} and I{add Person}). See
+    L{submitSelect}.
+    """
+    self.submitSelect('select_action', 'Base_doAction:method', label, value,
+                      **kw)
+
+  def submitCut(self):
+    """
+    Cut the previously selected objects.
+    """
+    self.submit(name='Folder_cut:method')
+
+  def submitCopy(self):
+    """
+    Copy the previously selected objects.
+    """
+    self.submit(name='Folder_copy:method')
+
+  def submitPaste(self):
+    """
+    Paste the previously selected objects.
+    """
+    self.submit(name='Folder_paste:method')
+
+  def submitPrint(self):
+    """
+    Print the previously selected objects. Use the class attribute
+    rather than the name as the latter is dependent on the context.
+    """
+    self.submit(class_attribute='print')
+
+  def submitReport(self):
+    """
+    Create a report. Use the class attribute rather than the name as
+    the latter is dependent on the context.
+    """
+    self.submit(class_attribute='report')
+
+  def submitNew(self):
+    """
+    Create a new object.
+    """
+    self.submit(name='Folder_create:method')
+
+  def submitDelete(self):
+    """
+    Delete the previously selected objects.
+    """
+    self.submit(name='Folder_deleteObjectList:method')
+
+  def submitSave(self):
+    """
+    Save the previously selected objects.
+    """
+    self.submit(name='Base_edit:method')
+
+  def submitShow(self):
+    """
+    Show the previously selected objects.
+    """
+    self.submit(name='Folder_show:method')
+
+  def submitFilter(self):
+    """
+    Filter the objects.
+    """
+    self.submit(name='Folder_filter:method')
+
+  def submitAction(self):
+    """
+    Select/unselect objects.
+    """
+    self.submit(name='Base_doSelect:method')
+
+  def submitSelectWorkflow(self, label=None, value=None,
+                           script_id='viewWorkflowActionDialog',
+                           maximum_attempt_number=1, sleep_between_attempt=0,
+                           **kw):
+    """
+    Select and submit a workflow action, given either by its label
+    (such as I{Create User}) or value (such as I{create_user_action}
+    in I{/Person_viewCreateUserActionDialog?workflow_action=create_user_action},
+    with C{script_id=Person_viewCreateUserActionDialog}). See L{submitSelect}.
+
+    When validating an object, L{submitDialogConfirm} allows to
+    perform the validation required on the next page.
+
+    As the Workflow action may not be available yet, it is possible to set the
+    maximum number of attempts and the sleep duration between each attempt.
+
+    @param script_id: Script identifier
+    @type script_id: str
+    @param maximum_attempt_number: Number of attempts before failing
+    @type maximum_attempt_number: int
+    @param sleep_between_attempt: Sleep N seconds between attempts
+    @type sleep_between_attempt: int
+    """
+    def tryLegacyAndNew():
+      try:
+        self.browser.mainForm.submitSelect(
+          'select_action', 'Base_doAction:method', label,
+          value and '%s?workflow_action=%s' % (script_id, value), **kw)
+
+      except LookupError:
+        self.browser.mainForm.submitSelect(
+          'select_action', 'Base_doAction:method', label,
+          value and '%s?field_my_workflow_action=%s' % (script_id, value), **kw)
+    
+    if maximum_attempt_number == 1:
+      tryLegacyAndNew()
+    else:
+      current_attempt_number = 1
+      while True:
+        try:
+          tryLegacyAndNew()
+        except LookupError:
+          if current_attempt_number == maximum_attempt_number:
+            raise
+
+          current_attempt_number += 1
+          time.sleep(sleep_between_attempt)
+        else:
+          break
+
+      return (current_attempt_number - 1) * sleep_between_attempt
+
+  def submitDialogCancel(self):
+    """
+    Cancel the dialog action. A dialog is showed when validating a
+    workflow or deleting an object for example.
+    """
+    self.submit(name='Base_cancel:method')
+
+  def submitDialogUpdate(self):
+    """
+    Update the dialog action. A dialog may contain a button to update
+    the form before confirming it. See L{submitDialogConfirm} as well.
+    """
+    self.submit(name='Base_showUpdateDialog:method')
+
+  def submitDialogConfirm(self):
+    """
+    Confirm the dialog action. A dialog is showed when validating a
+    workflow or deleting an object for example.
+
+    @todo: Specifying index is kind of ugly (there is C{dummy} field
+           with the same name though)
+    """
+    self.submit(name='Base_callDialogMethod:method')
+
+  def getListboxControl(self, line_number, column_number, cell_element_index=1,
+                        *args, **kwargs):
+    """
+    Get the control located at line and column numbers (both starting
+    from 1), excluding hidden control and those whose class is hidden
+    too. The position of a cell from a column or line number can be
+    obtained through calling
+    L{erp5.util.test_browser.browser.Browser.getListboxPosition}.
+
+    Also, there may be several elements within a cell, thus
+    C{cell_element_index} allows to select which one to get (starting
+    from 1).
+
+    @param line_number: Line number of the field
+    @type line_number: int
+    @param column_number: Column number of the field
+    @type column_number: int
+    @param cell_element_index: Index of the control to be selected in the cell
+    @type cell_element_index: int
+    @param args: positional arguments given to the parent C{getControl}
+    @type args: list
+    @param kwargs: keyword arguments given to the parent C{getControl}
+    @type kwargs: dict
+    @return: The control found at the given line and column numbers
+    @rtype: L{zope.testbrowser.interfaces.IControl}
+
+    @raise LookupError: No control could be found at the given
+                        position and cell indexes
+    """
+    if self.browser._is_legacy_listbox:
+      listbox_basic_xpath_str = self.browser._legacy_listbox_table_xpath_str
+    else:
+      listbox_basic_xpath_str = self.browser._listbox_table_xpath_str
+
+    if line_number <= 2:
+      relative_line_number = line_number
+
+      if self.browser._is_legacy_listbox:
+        column_type = 'td'
+      else:
+        column_type = 'th'
+    else:
+      if self.browser._is_legacy_listbox:
+        relative_line_number = line_number
+      else:
+        relative_line_number = line_number - 2
+
+      column_type = 'td'
+
+    xpath_str = '%s//tr[%d]//%s[%d]/*[not(@type="hidden") and ' \
+        'not(contains(@class, "hidden"))][%d]' % \
+        (listbox_basic_xpath_str,
+         relative_line_number,
+         column_type,
+         column_number,
+         cell_element_index)
+
+    # xpath() method always return a list even if there is only one element
+    element_list = self.browser.etree.xpath(xpath_str)
+
+    try:
+      input_element = element_list[0]
+      input_name = input_element.get('name')
+    except (IndexError, AttributeError):
+      input_element = input_name = None
+
+    if input_element is None or not input_name:
+      raise LookupError("Could not find control in listbox cell %dx%d (index=%d)" %\
+                          (line_number, column_number, cell_element_index))
+
+    control = self.getControl(name=input_element.get('name'), *args, **kwargs)
+
+    # If this is a list control (radio button, checkbox or select
+    # control), then get the item from its value
+    if isinstance(control, ListControl):
+      control = control.getControl(value=input_element.get('value'))
+
+    return control
+
+
+from zope.testbrowser.browser import Link
+
+class LinkWithTime(Link):
+  """
+  Only define to wrap click methods to measure the time spent
+  """
+  __metaclass__ = measurementMetaClass(prefix='click')

erp5/util/test_browser/examples/testAddPerson.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from erp5.util.test_browser.browser import Browser
+
+ITERATION = 20
+
+def benchmarkAddPerson(iteration_counter, result_dict):
+  """
+  Benchmark adding a person.
+  """
+  # Create a browser instance
+  browser = Browser('http://localhost:18080/', 'erp5',
+                    username='zope', password='zope')
+
+  # Open ERP5 homepage
+  browser.open()
+
+  # Go to Persons module (person_module)
+  browser.mainForm.submitSelectModule(value='/person_module')
+
+  # Create a new person and record the time elapsed in seconds
+  result_dict.setdefault('Create', []).append(
+    browser.mainForm.submitNew())
+
+  # Check whether it has been successfully created
+  assert browser.getTransitionMessage() == 'Object created.'
+
+  # Fill the first and last name of the newly created person
+  browser.mainForm.getControl(name='field_my_first_name').value = 'Foo%d' % \
+      iteration_counter
+
+  browser.mainForm.getControl(name='field_my_last_name').value = 'Bar%d' % \
+      iteration_counter
+
+  # Submit the changes, record the time elapsed in seconds
+  result_dict.setdefault('Save', []).append(
+    browser.mainForm.submitSave())
+
+  # Check whether the changes have been successfully updated
+  assert browser.getTransitionMessage() == 'Data updated.'
+
+  # Validate the person (as the workflow action may not be available yet, try
+  # 5 times and sleep 5s between each attempts before failing) and record
+  # time spent on confirmation
+  browser.mainForm.submitSelectWorkflow(value='validate_action',
+                                        maximum_attempt_number=5,
+                                        sleep_between_attempt=5)
+  result_dict.setdefault('Validate', []).append(
+    browser.mainForm.submitDialogConfirm())
+
+  # Check whether it has been successfully validated
+  assert browser.getTransitionMessage() == 'Status changed.'
+
+
+  ## Go to the new person from the Persons module, showing how to use
+  ## listbox API
+  # Go to Persons module first (person_module)
+  browser.mainForm.submitSelectModule(value='/person_module')
+
+  # Select all the persons whose Usual Name starts with Foo
+  browser.mainForm.getListboxControl(2, 2).value = 'Foo%'
+
+  result_dict.setdefault('Filter', []).append(
+    browser.mainForm.submit())
+
+  # Get the line number
+  line_number = browser.getListboxPosition("Foo%(counter)d Bar%(counter)d" % \
+                                             {'counter': iteration_counter},
+                                           column_number=2)
+
+  # From the column and line_number, we can now get the Link instance
+  link = browser.getListboxLink(line_number=line_number, column_number=2)
+
+  # Click on the link
+  link.click()
+
+  assert browser.mainForm.getControl(name='field_my_first_name').value == \
+      'Foo%d' % iteration_counter
+
+if __name__ == '__main__':
+  # Run benchmarkAddPerson ITERATION times and compute the average time it
+  # took for each operation
+  result_dict = {}
+  counter = 0
+  while counter != ITERATION:
+    benchmarkAddPerson(counter, result_dict)
+    counter += 1
+
+  for title, time_list in result_dict.iteritems():
+    print "%s: %.4fs" % (title, float(sum(time_list)) / ITERATION)

setup.py

@@ -39,6 +39,10 @@ setup(name=name,
       ],
       extras_require={
         'testnode': ['slapos.core', 'xml_marshaller'],
+        'test_browser': ['zope.testbrowser >= 3.11.1', 'z3c.etestbrowser'],
+        'benchmark': [name+'[test_browser]'],
+        'benchmark-report': [name+'[benchmark]', 'matplotlib', 'numpy'],
+        'scalability_tester': [name+'[benchmark]', 'slapos.tool.nosqltester'],
       },
       zip_safe=True,
       packages=find_packages(),
@@ -46,6 +50,9 @@ setup(name=name,
       entry_points={
         'console_scripts': [
           'testnode = erp5.util.testnode:main [testnode]',
+          'performance_tester_erp5 = erp5.util.benchmark.performance_tester:main [benchmark]',
+          'scalability_tester_erp5 = erp5.util.benchmark.scalability_tester:main [scalability_tester]',
+          'generate_erp5_tester_report = erp5.util.benchmark.report:generateReport [benchmark-report]',
         ],
       }
     )