ScalabilityTestRunner.py 8.97 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39
##############################################################################
#
# Copyright (c) 2011 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly advised to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################
from datetime import datetime,timedelta
import os
import subprocess
import sys
import time
import glob
import SlapOSControler
import json
import time
import shutil
import logging
import string
import random
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
40
import testnodeUtils
41 42 43 44
from ProcessManager import SubprocessError, ProcessManager, CancellationError
from subprocess import CalledProcessError
from Updater import Updater
from erp5.util import taskdistribution
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
45 46
# for dummy slapos answer
import signal
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
47

48

49 50 51
class ScalabilityTestRunner():
  def __init__(self, testnode):
    self.testnode =  testnode
52 53 54 55
    self.slapos_controler = SlapOSControler.SlapOSControler(
                                  self.testnode.working_directory,
                                  self.testnode.config,
                                  self.testnode.log)
56 57 58
    # Create the slapos account configuration file and dir
    key = self.testnode.test_suite_portal.getSlaposAccountKey()
    certificate = self.testnode.test_suite_portal.getSlaposAccountCertificate()
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
59 60
    self.slapos_controler.createSlaposConfigurationFileAccount(key,certificate,
                                    self.testnode.config, self.testnode.log)
Benjamin Blanc's avatar
Benjamin Blanc committed
61 62
    # {'COMPX' : ['soft_path1.cfg', 'soft_path2.cfg'],
    #  'COMPY' : ['soft_path1.cfg'], ... }
63
    self.remaining_software_installation_grid = {}
Benjamin Blanc's avatar
Benjamin Blanc committed
64
    # Protection to prevent installation of softwares after checking
Benjamin Blanc's avatar
Benjamin Blanc committed
65
    self.authorize_supply = True
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
66 67 68 69
    
    # Used to simulate SlapOS answer
    self.last_slapos_answer = []
    
70 71
  def _prepareSlapOS(self, software_path, computer_guid, create_partition=0):
    # create_partition is kept for compatibility
72
    """
Benjamin Blanc's avatar
Benjamin Blanc committed
73
    A proxy to supply : Install a software on a specific node
Benjamin Blanc's avatar
Benjamin Blanc committed
74
    """
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
75
    self.testnode.log("testnode, supply : %s %s", software_path, computer_guid)
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
76
    if self.authorize_supply :
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
77
      if not computer_guid in self.remaining_software_installation_grid:
Benjamin Blanc's avatar
Benjamin Blanc committed
78 79 80
        # Add computer_guid to the grid if it isn't
        self.remaining_software_installation_grid[computer_guid] = []
      self.remaining_software_installation_grid[computer_guid].append(software_path)
Benjamin Blanc's avatar
Benjamin Blanc committed
81
      self.slapos_controler.supply(software_path, computer_guid)
82
      # Here make a request via slapos controler ?
Benjamin Blanc's avatar
Benjamin Blanc committed
83 84
      return {'status_code' : 0}                                          
    else:
Benjamin Blanc's avatar
Benjamin Blanc committed
85
      raise ValueError("Too late to supply now. ('self.authorize_supply' is False)")
Benjamin Blanc's avatar
Benjamin Blanc committed
86
      
87
  def prepareSlapOSForTestNode(self, test_node_slapos=None):
88
    """
Benjamin Blanc's avatar
Benjamin Blanc committed
89
    We will build slapos software needed by the testnode itself,
90
    """
Benjamin Blanc's avatar
Test  
Benjamin Blanc committed
91 92
    if self.testnode.test_suite_portal.isValidatedMaster(
                           self.testnode.config['test_node_title']):
Benjamin Blanc's avatar
Benjamin Blanc committed
93
      pass
Benjamin Blanc's avatar
Benjamin Blanc committed
94 95
#      software_path_list = []
#      software_path_list.append(self.testnode.config.get("software_list"))
Benjamin Blanc's avatar
Benjamin Blanc committed
96
    return {'status_code' : 0} 
Benjamin Blanc's avatar
Benjamin Blanc committed
97

Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
98 99 100
  # For dummy slapos answer
  # Press ctrl+c to simulate an (positive) answer from sapos master
  def _getSignal(self, signal, frame):
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
101
    self.testnode.log("Dummy SlapOS Master answer received.")
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
102 103 104
    self.last_slapos_answer.append(True)
  def _prepareDummySlapOSAnswer(self):
    signal.signal(signal.SIGINT, self._getSignal)
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
105
  def _comeBackFromDummySlapOS(self):
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
106
    signal.signal(signal.SIGINT, signal.SIG_IGN)
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
107 108 109 110 111 112 113
  def simulateSlapOSAnswer(self):
    if len(self.last_slapos_answer)==0:
      return False
    else:
      return self.last_slapos_answer.pop()
  # /For dummy slapos answer
    
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
114
  def isSoftwareReleaseReady(self, software_url, computer_guid):
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
115 116 117 118
    """
    Return true if the specified software on the specified node is installed.
    This method should communicates with SlapOS Master.
    """
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
119 120 121
    # TODO : implement -> communication with SlapOS master
    # this simulate a SlapOS answer
    return self.simulateSlapOSAnswer()
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
122
      
Benjamin Blanc's avatar
Benjamin Blanc committed
123
  def remainSoftwareToInstall(self):
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
124
    """
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
125
    Return True if it remains softwares to install, otherwise return False
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
126 127
    """
    # Remove from grid installed software entries
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
128
    for computer_guid,v in self.remaining_software_installation_grid.items():
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
129
      for software_url in v:
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
130
        if self.isSoftwareReleaseReady(software_url, computer_guid):
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
131 132 133
          self.remaining_software_installation_grid[computer_guid].remove(software_url)
        if len(self.remaining_software_installation_grid[computer_guid])==0:
          del self.remaining_software_installation_grid[computer_guid]
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
134 135
    # Not empty grid means that all softwares are not installed
    return len(self.remaining_software_installation_grid) > 0
Benjamin Blanc's avatar
Benjamin Blanc committed
136
          
137
  def prepareSlapOSForTestSuite(self, node_test_suite):
138
    """
Benjamin Blanc's avatar
Benjamin Blanc committed
139
    Install testsuite softwares
140
    """
Benjamin Blanc's avatar
Benjamin Blanc committed
141
    # Define how many time this method can take
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
142
    max_time = 3600*10*1.0 # 10 hours
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
143
    interval_time = 30
Benjamin Blanc's avatar
Benjamin Blanc committed
144
    start_time = time.time()
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
145
    # Only master testnode must order software installation
Benjamin Blanc's avatar
Benjamin Blanc committed
146
    if self.testnode.test_suite_portal.isValidatedMaster(
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
147 148 149 150 151 152 153 154 155 156 157 158 159 160
            self.testnode.config['test_node_title']):
      # Get from ERP5 Master the configuration of the cluster for the test
      test_configuration = testnodeUtils.deunicodeData(
          json.loads(self.testnode.test_suite_portal.generateConfiguration(
                      node_test_suite.test_suite_title)
                    )
        )
      self.involved_nodes_computer_guid = test_configuration['involved_nodes_computer_guid']
      self.launchable = test_configuration['launchable']
      self.error_message = test_configuration['error_message']
      if not self.launchable:
        self.testnode.log("Test suite %s is not actually launchable with \
  the current cluster configuration." %(node_test_suite.test_suite_title,))
        self.testnode.log("ERP5 Master indicates : %s" %(self.error_message,))
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
161
        # error : wich code to return ?
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
162 163 164 165 166 167 168 169 170 171 172 173 174
        return {'status_code' : 1}
      involved_nodes_computer_guid = test_configuration['involved_nodes_computer_guid']
      configuration_list = test_configuration['configuration_list']
      launcher_nodes_computer_guid = test_configuration['launcher_nodes_computer_guid']
      software_path_list = []
      # Here add the ipv6 url reachable from master profile
      software_path_list.append("http://foo.bar/It_is_a_test_for_scalability_test/My_unreachable_profile.cfg")
      # Ask for softwares installation
      for software_path in software_path_list:
        for computer_guid in self.involved_nodes_computer_guid:
          self._prepareSlapOS(software_path, computer_guid) 
      # From the line below we would not supply any more softwares
      self.authorize_supply = False
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
175
      # TODO : remove the line below wich simulate an answer from slapos master
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
176
      self._prepareDummySlapOSAnswer()
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
177 178
      # Waiting until all softwares are installed
      while ( self.remainSoftwareToInstall() 
179
         and (max_time > (time.time()-start_time))):
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
180
        self.testnode.log("Master testnode is waiting\
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
181
  for the end of all software installation (for %ss).",
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
182
          str(int(time.time()-start_time)))
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
183
        time.sleep(interval_time)
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
184
      # We were wainting for too long time, that's a failure.
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
185 186
      # TODO : remove the line below wich simulate an answer from slapos master
      self._comeBackFromDummySlapOS()
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
187 188
      if self.remainSoftwareToInstall() :
        return {'status_code' : 1}
189
    return {'status_code' : 0}
190 191

  def _cleanUpNodesInformation(self):
Benjamin Blanc's avatar
Benjamin Blanc committed
192 193
    self.involved_nodes_computer_guid = []
    self.launcher_nodes_computer_guid = []
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
194 195
    self.remaining_software_installation_grid = {}
    self.authorize_supply = True
196

197 198
  def runTestSuite(self, node_test_suite, portal_url, log=None):
    # TODO : write code
199 200
    SlapOSControler.createFolder(node_test_suite.test_suite_directory,
                                 clean=True)
Benjamin Blanc's avatar
Benjamin Blanc committed
201
    # create ResultLine for each loop    
202
    pass
203

Benjamin Blanc's avatar
Benjamin Blanc committed
204
  def getRelativePathUsage(self):
Benjamin Blanc's avatar
Up  
Benjamin Blanc committed
205 206 207 208
    """
    Used by the method testnode.constructProfile() to know
    if the software.cfg have to use relative path or not.
    """
Benjamin Blanc's avatar
Benjamin Blanc committed
209
    return True