Commit d5355eb0 authored by Roque's avatar Roque Committed by Klaus Wölfel

testnode: review scalability tests up to the creation of slapos instance

- Most of the communication with slapos master is done using the new Slapos Master Communicator instead of Slapos Controller:
  * supply, request, check instance status, check software status.
- Refactor of related methods (waitInstance, isSoftwareReleaseReady, etc.)
- Initialization of communicator: initialize connection to SlapOS usign key and certificate
- General refactor and cleanup: some unnecesary sentences, comments and methods were removed.
parent 50a3a319
...@@ -45,6 +45,7 @@ from Updater import Updater ...@@ -45,6 +45,7 @@ from Updater import Updater
from erp5.util import taskdistribution from erp5.util import taskdistribution
# for dummy slapos answer # for dummy slapos answer
import signal import signal
import slapos.slap
# max time to instance changing state: 2 hour # max time to instance changing state: 2 hour
MAX_INSTANCE_TIME = 60*60*2 MAX_INSTANCE_TIME = 60*60*2
...@@ -52,6 +53,8 @@ MAX_INSTANCE_TIME = 60*60*2 ...@@ -52,6 +53,8 @@ MAX_INSTANCE_TIME = 60*60*2
MAX_CREATION_INSTANCE_TIME = 60*10 MAX_CREATION_INSTANCE_TIME = 60*10
# max time for a test: 1 hour # max time for a test: 1 hour
MAX_TEST_CASE_TIME = 60*60 MAX_TEST_CASE_TIME = 60*60
# max time to prepare SlapOS for testsuite (software installation, instances requests, etc.)
MAX_PREPARE_TEST_SUITE = 3600*10*1.0 # 10 hours
class ScalabilityTestRunner(): class ScalabilityTestRunner():
def __init__(self, testnode): def __init__(self, testnode):
...@@ -103,8 +106,7 @@ class ScalabilityTestRunner(): ...@@ -103,8 +106,7 @@ class ScalabilityTestRunner():
self.log("testnode, supply : %s %s", software_path, computer_guid) self.log("testnode, supply : %s %s", software_path, computer_guid)
if self.authorize_supply : if self.authorize_supply :
self.remaining_software_installation_dict[computer_guid] = software_path self.remaining_software_installation_dict[computer_guid] = software_path
self.slapos_controler.supply(software_path, computer_guid) self.slapos_communicator._supply()
# Here make a request via slapos controler ?
return {'status_code' : 0} return {'status_code' : 0}
else: else:
raise ValueError("Too late to supply now. ('self.authorize_supply' is False)") raise ValueError("Too late to supply now. ('self.authorize_supply' is False)")
...@@ -114,12 +116,14 @@ class ScalabilityTestRunner(): ...@@ -114,12 +116,14 @@ class ScalabilityTestRunner():
""" """
Generate an instance title using various parameter Generate an instance title using various parameter
TODO : add some verification (to don't use unexisting variables) TODO : add some verification (to don't use unexisting variables)
ROQUE XXX TODO: for now, the name is always the same. The instance is not being destroyed yet.
""" """
instance_title = "Scalability-" instance_title = "Scalability-"
instance_title += "("+test_suite_title+")-" instance_title += "("+test_suite_title+")-"
instance_title += str(self.involved_nodes_computer_guid).replace("'","") instance_title += str(self.involved_nodes_computer_guid).replace("'","")
instance_title += "-"+str(datetime.datetime.now().isoformat())+"-" instance_title += "-2" # XXX hack to force instance creation. Will be removed.
instance_title += "timestamp="+str(time.time()) #instance_title += "-"+str(datetime.datetime.now().isoformat())+"-"
#instance_title += "timestamp="+str(time.time())
return instance_title return instance_title
def _generateInstanceXML(self, software_configuration, def _generateInstanceXML(self, software_configuration,
...@@ -127,8 +131,7 @@ class ScalabilityTestRunner(): ...@@ -127,8 +131,7 @@ class ScalabilityTestRunner():
""" """
Generate a complete scalability instance XML configuration Generate a complete scalability instance XML configuration
""" """
config_cluster = software_configuration.copy() config = software_configuration.copy()
config = {'cluster':config_cluster}
config.update({'scalability-launcher-computer-guid':self.launcher_nodes_computer_guid[0]}) config.update({'scalability-launcher-computer-guid':self.launcher_nodes_computer_guid[0]})
config.update({'scalability-launcher-title':'MyTestNodeTitle'}) config.update({'scalability-launcher-title':'MyTestNodeTitle'})
config.update({'test-result-path':test_result.test_result_path}) config.update({'test-result-path':test_result.test_result_path})
...@@ -143,13 +146,11 @@ class ScalabilityTestRunner(): ...@@ -143,13 +146,11 @@ class ScalabilityTestRunner():
Create scalability instance Create scalability instance
""" """
if self.authorize_request: if self.authorize_request:
self.log("testnode, request : %s", instance_title)
config = self._generateInstanceXML(software_configuration, config = self._generateInstanceXML(software_configuration,
test_result, test_suite) test_result, test_suite)
self.log("testnode, request : %s", instance_title) request_kw = {"partition_parameter_kw": {"_" : json.dumps(config)} }
config = json.dumps(config) self.slapos_communicator._request(SlapOSMasterCommunicator.INSTANCE_STATE_STARTED, instance_title, request_kw)
self.slapos_controler.request(instance_title, software_path,
"test", {"_" : config},
self.launcher_nodes_computer_guid[0])
self.authorize_request = False self.authorize_request = False
return {'status_code' : 0} return {'status_code' : 0}
else: else:
...@@ -190,9 +191,8 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -190,9 +191,8 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
Return true if the specified software on the specified node is installed. Return true if the specified software on the specified node is installed.
This method should communicates with SlapOS Master. This method should communicates with SlapOS Master.
""" """
# TODO : implement -> communication with SlapOS master self.log("Current software state: " + str(self.slapos_communicator._getSoftwareState()))
# this simulate a SlapOS answer return self.slapos_communicator._getSoftwareState() == SlapOSMasterCommunicator.SOFTWARE_STATE_INSTALLED
return self.simulateSlapOSAnswer()
def remainSoftwareToInstall(self): def remainSoftwareToInstall(self):
""" """
...@@ -207,23 +207,20 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -207,23 +207,20 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
def _updateInstanceXML(self, software_configuration, instance_title, def _updateInstanceXML(self, software_configuration, instance_title,
test_result, test_suite): test_result, test_suite):
""" self.log("testnode, updateInstanceXML : %s", instance_title)
Just a proxy to SlapOSControler.updateInstanceXML.
"""
config = self._generateInstanceXML(software_configuration, config = self._generateInstanceXML(software_configuration,
test_result, test_suite) test_result, test_suite)
config = json.dumps(config) request_kw = {"partition_parameter_kw": {"_" : json.dumps(config)} }
self.log("testnode, updateInstanceXML : %s", instance_title) self.slapos_communicator._request(SlapOSMasterCommunicator.INSTANCE_STATE_STARTED, instance_title, request_kw)
self.slapos_controler.updateInstanceXML(instance_title, {"_" : config})
return {'status_code' : 0} return {'status_code' : 0}
def _waitInstance(self, instance_title, state, max_time=MAX_INSTANCE_TIME): def _waitInstance(self, instance_title, state, max_time=MAX_INSTANCE_TIME):
""" """
Wait for 'max_time' an instance specific state Wait for 'max_time' an instance specific state
""" """
self.log("Wait for instance state: %s" %state) self.log("Waiting for instance state: %s" %state)
start_time = time.time() start_time = time.time()
while (not self.slapos_communicator.isHostingSubscriptionReady(instance_title, state) while (not self.slapos_communicator._getInstanceState() == state
and (max_time > (time.time()-start_time))): and (max_time > (time.time()-start_time))):
self.log("Instance(s) not in %s state yet." % state) self.log("Instance(s) not in %s state yet." % state)
time.sleep(15) time.sleep(15)
...@@ -231,11 +228,8 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -231,11 +228,8 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
error_message = "Instance '%s' not '%s' after %s seconds" %(instance_title, state, str(time.time()-start_time)) error_message = "Instance '%s' not '%s' after %s seconds" %(instance_title, state, str(time.time()-start_time))
self.log(error_message) self.log(error_message)
self.log("Do you use instance state propagation in your project?") self.log("Do you use instance state propagation in your project?")
self.log("Instance '%s' will be stopped and test avorted." %instance_title) self.log("Instance '%s' will be stopped and test aborted." %instance_title)
# What if we wanted to stop ? self.slapos_communicator._request(SlapOSMasterCommunicator.INSTANCE_STATE_STOPPED)
self.slapos_controler.stopInstance(instance_title)
# XXX: _waitInstance call here ? recursive call ?
# XXX: sleep 60 seconds.
time.sleep(60) time.sleep(60)
raise ValueError(error_message) raise ValueError(error_message)
self.log("Instance correctly '%s' after %s seconds." %(state, str(time.time()-start_time))) self.log("Instance correctly '%s' after %s seconds." %(state, str(time.time()-start_time)))
...@@ -244,10 +238,11 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -244,10 +238,11 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
""" """
Wait for 'max_time' the instance creation Wait for 'max_time' the instance creation
""" """
self.log("Wait for instance creation") self.log("Waiting for instance creation...")
start_time = time.time() start_time = time.time()
while ( not self.slapos_communicator.isRegisteredHostingSubscription(instance_title) \ while (not self.slapos_communicator.isInstanceRequested(instance_title) \
and (max_time > (time.time()-start_time)) ): and (max_time > (time.time()-start_time)) ):
self.log("Instance not ready yet. Sleeping 5 sec.")
time.sleep(5) time.sleep(5)
if (time.time()-start_time) > max_time: if (time.time()-start_time) > max_time:
raise ValueError("Instance '%s' not found after %s seconds" %(instance_title, max_time)) raise ValueError("Instance '%s' not found after %s seconds" %(instance_title, max_time))
...@@ -257,37 +252,52 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -257,37 +252,52 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
""" """
Install testsuite softwares Install testsuite softwares
""" """
self.log('prepareSlapOSForTestSuite') self.log('Preparing SlapOS for Test Suite...')
# Define how many time this method can take max_time = MAX_PREPARE_TEST_SUITE
max_time = 3600*10*1.0 # 10 hours
interval_time = 60 interval_time = 60
start_time = time.time() start_time = time.time()
# Create a communicator with slapos
self.log("creating SlapOs Master communicator...") # Initialize communication with slapos
self.slapos_communicator = SlapOSMasterCommunicator.SlapOSMasterCommunicator( slap = slapos.slap.slap()
self.cert_path, retry = 0
while True:
# wait until _hateoas_navigator is loaded.
if retry > 100:
break
slap.initializeConnection(self.slapos_url,
self.key_path, self.key_path,
self.log, self.cert_path,
self.slapos_api_rest_url) timeout=120,
slapgrid_rest_uri=self.slapos_api_rest_url)
if getattr(slap, '_hateoas_navigator', None) is None:
retry += 1
self.log("Fail to load _hateoas_navigator waiting a bit and retry.")
time.sleep(30)
else:
break
if getattr(slap, '_hateoas_navigator', None) is None:
raise ValueError("Fail to load _hateoas_navigator")
hateoas = getattr(slap, '_hateoas_navigator', None)
supply = slap.registerSupply()
order = slap.registerOpenOrder()
# Only master testnode must order software installation # Only master testnode must order software installation
if self.testnode.test_suite_portal.isMasterTestnode( if self.testnode.test_suite_portal.isMasterTestnode(
self.testnode.config['test_node_title']): self.testnode.config['test_node_title']):
# Get from ERP5 Master the configuration of the cluster for the test # Get from ERP5 Master the configuration of the cluster for the test
test_configuration = Utils.deunicodeData( test_configuration = Utils.deunicodeData(
json.loads(self.testnode.test_suite_portal.generateConfiguration( json.loads(self.testnode.test_suite_portal.generateConfiguration(
node_test_suite.test_suite_title) node_test_suite.test_suite_title)))
)
)
self.involved_nodes_computer_guid = test_configuration['involved_nodes_computer_guid'] self.involved_nodes_computer_guid = test_configuration['involved_nodes_computer_guid']
self.launchable = test_configuration['launchable'] self.launchable = test_configuration['launchable']
self.error_message = test_configuration['error_message'] self.error_message = test_configuration['error_message']
self.randomized_path = test_configuration['randomized_path'] self.randomized_path = test_configuration['randomized_path']
# Avoid the test if it is not launchable
if not self.launchable: if not self.launchable:
self.log("Test suite %s is not actually launchable with \ self.log("Test suite %s is not actually launchable with \
the current cluster configuration." %(node_test_suite.test_suite_title,)) the current cluster configuration." %(node_test_suite.test_suite_title,))
self.log("ERP5 Master indicates : %s" %(self.error_message,)) self.log("ERP5 Master indicates : %s" %(self.error_message,))
# error : wich code to return ?
return {'status_code' : 1} return {'status_code' : 1}
involved_nodes_computer_guid = test_configuration['involved_nodes_computer_guid'] involved_nodes_computer_guid = test_configuration['involved_nodes_computer_guid']
...@@ -318,6 +328,9 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -318,6 +328,9 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
"https://","["+self.testnode.config['httpd_ip']+"]"+":"+self.testnode.config['httpd_software_access_port'], "https://","["+self.testnode.config['httpd_ip']+"]"+":"+self.testnode.config['httpd_software_access_port'],
self.randomized_path) self.randomized_path)
self.reachable_profile = os.path.join(self.reachable_address, "software.cfg") self.reachable_profile = os.path.join(self.reachable_address, "software.cfg")
# ROQUE XXX: below urls are hardcoded until we find ways to make buildout extending through unsafe https
self.reachable_address = "https://lab.nexedi.com/rporchetto/telecom/tree/master"
self.reachable_profile = "https://lab.nexedi.com/rporchetto/telecom/raw/master/software_release/software.cfg"
# Write the reachable address in the software.cfg file, # Write the reachable address in the software.cfg file,
# by replacing <obfuscated_url> occurences by the current reachable address. # by replacing <obfuscated_url> occurences by the current reachable address.
...@@ -337,13 +350,21 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -337,13 +350,21 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
# Ask for SR installation # Ask for SR installation
for computer_guid in self.involved_nodes_computer_guid: for computer_guid in self.involved_nodes_computer_guid:
self.slapos_communicator = SlapOSMasterCommunicator.SoftwareReleaseTester(
"SlaposMasterCommunicator",
self.log,
slap,
order,
supply,
self.reachable_profile,
computer_guid=computer_guid)
self._prepareSlapOS(self.reachable_profile, computer_guid) self._prepareSlapOS(self.reachable_profile, computer_guid)
# From the line below we would not supply any more softwares # From the line below we would not supply any more softwares
self.authorize_supply = False self.authorize_supply = False
# TODO : remove the line below wich simulate an answer from slapos master # TODO : remove the line below wich simulate an answer from slapos master
self._prepareDummySlapOSAnswer() self._prepareDummySlapOSAnswer()
# Waiting until all softwares are installed # Waiting until all softwares are installed
while ( self.remainSoftwareToInstall() while (self.remainSoftwareToInstall()
and (max_time > (time.time()-start_time))): and (max_time > (time.time()-start_time))):
self.log("Master testnode is waiting\ self.log("Master testnode is waiting\
for the end of all software installation (for %ss) PID=%s.", for the end of all software installation (for %ss) PID=%s.",
...@@ -355,68 +376,76 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -355,68 +376,76 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
# All softwares are not installed, however maxtime is elapsed, that's a failure. # All softwares are not installed, however maxtime is elapsed, that's a failure.
return {'status_code' : 1} return {'status_code' : 1}
self.authorize_request = True self.authorize_request = True
self.log("Softwares installed") self.log("Softwares installed.")
# Launch instance # Launch instance
self.instance_title = self._generateInstanceTitle(node_test_suite.test_suite_title) self.instance_title = self._generateInstanceTitle(node_test_suite.test_suite_title)
try: try:
self._createInstance(self.reachable_profile, configuration_list[0], self._createInstance(self.reachable_profile,
self.instance_title, node_test_suite.test_result, node_test_suite.test_suite) configuration_list[0],
self.instance_title,
node_test_suite.test_result,
node_test_suite.test_suite)
self.log("Scalability instance requested.") self.log("Scalability instance requested.")
except: except:
self.log("Unable to launch instance") self.log("Unable to launch instance")
raise ValueError("Unable to launch instance") raise ValueError("Unable to launch instance")
self.log("Waiting for instance creation..")
self._waitInstanceCreation(self.instance_title) self._waitInstanceCreation(self.instance_title)
return {'status_code' : 0} return {'status_code' : 0}
return {'status_code' : 1} return {'status_code' : 1}
def runTestSuite(self, node_test_suite, portal_url): def runTestSuite(self, node_test_suite, portal_url):
if not self.launchable: if not self.launchable:
self.log("Current test_suite is not actually launchable.") self.log("Current test_suite is not actually launchable.")
return {'status_code' : 1} # Unable to continue due to not realizable configuration return {'status_code' : 1} # Unable to continue due to not realizable configuration
configuration_list = node_test_suite.configuration_list configuration_list = node_test_suite.configuration_list
test_list = range(0, len(configuration_list)) test_list = range(0, len(configuration_list))
# create test_result
test_result_proxy = self.testnode.portal.createTestResult( test_result_proxy = self.testnode.portal.createTestResult(
node_test_suite.revision, test_list, node_test_suite.revision, test_list,
self.testnode.config['test_node_title'], self.testnode.config['test_node_title'],
True, node_test_suite.test_suite_title, True, node_test_suite.test_suite_title,
node_test_suite.project_title) node_test_suite.project_title)
self.log("Test Result created.")
count = 0 count = 0
error_message = None error_message = None
self._waitInstance(self.instance_title, SlapOSMasterCommunicator.INSTANCE_STATE_STARTED)
# Each cluster configuration are tested # Each cluster configuration are tested
for configuration in configuration_list: for configuration in configuration_list:
# First configuration doesn't need XML configuration update. # First configuration doesn't need XML configuration update.
if count > 0: if count > 0:
# Stop instance self.slapos_communicator._request(SlapOSMasterCommunicator.INSTANCE_STATE_STOPPED)
self.slapos_controler.stopInstance(self.instance_title) self._waitInstance(self.instance_title, SlapOSMasterCommunicator.INSTANCE_STATE_STOPPED)
self._waitInstance(self.instance_title, 'stopped')
# Update instance XML configuration
self._updateInstanceXML(configuration, self.instance_title, self._updateInstanceXML(configuration, self.instance_title,
node_test_suite.test_result, node_test_suite.test_suite) node_test_suite.test_result, node_test_suite.test_suite)
self._waitInstance(self.instance_title, 'started') self._waitInstance(self.instance_title, SlapOSMasterCommunicator.INSTANCE_STATE_STARTED)
# Start instance self.slapos_communicator._request(SlapOSMasterCommunicator.INSTANCE_STATE_STARTED)
self.slapos_controler.startInstance(self.instance_title)
'''
# XXX: Dirty hack used to force haproxy to restart in time # ROQUE: for now, this old hack is not needed
# with all zope informations. # XXX: Dirty hack used to force haproxy to restart in time with all zope informations.
self._waitInstance(self.instance_title, 'started') self._waitInstance(self.instance_title, SlapOSMasterCommunicator.INSTANCE_STATE_STARTED)
self.slapos_controler.stopInstance(self.instance_title) self.slapos_communicator._request(SlapOSMasterCommunicator.INSTANCE_STATE_STOPPED)
self._waitInstance(self.instance_title, 'stopped') self._waitInstance(self.instance_title, SlapOSMasterCommunicator.INSTANCE_STATE_STOPPED)
self.slapos_controler.startInstance(self.instance_title) self.slapos_communicator._request(SlapOSMasterCommunicator.INSTANCE_STATE_STARTED)
########################################################## ##########################################################
'''
self._waitInstance(self.instance_title, SlapOSMasterCommunicator.INSTANCE_STATE_STARTED)
self.log("[DEBUG] INSTANCE CORRECTLY STARTED")
self._waitInstance(self.instance_title, 'started') # ROQUE XXX : for debug
if True:
self.log("RETURN FOR DEBUG")
return {'status_code' : 0}
# Start only the current test # Start only the current test
exclude_list=[x for x in test_list if x!=test_list[count]] exclude_list=[x for x in test_list if x!=test_list[count]]
count += 1 count += 1
test_result_line_proxy = test_result_proxy.start(exclude_list) test_result_line_proxy = test_result_proxy.start(exclude_list)
#
if test_result_line_proxy == None : if test_result_line_proxy == None :
error_message = "Test case already tested." error_message = "Test case already tested."
break break
...@@ -446,8 +475,8 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),)) ...@@ -446,8 +475,8 @@ late a SlapOS (positive) answer." %(str(os.getpid()),str(os.getpid()),))
break break
# Stop current instance # Stop current instance
self.slapos_controler.stopInstance(self.instance_title) self.slapos_communicator._request(SlapOSMasterCommunicator.INSTANCE_STATE_STOPPED)
self._waitInstance(self.instance_title, 'stopped') self._waitInstance(self.instance_title, SlapOSMasterCommunicator.INSTANCE_STATE_STOPPED)
# Delete old instances # Delete old instances
self._cleanUpOldInstance() self._cleanUpOldInstance()
......
...@@ -375,7 +375,7 @@ shared = true ...@@ -375,7 +375,7 @@ shared = true
if my_test_type == 'UnitTest': if my_test_type == 'UnitTest':
runner = UnitTestRunner(node_test_suite) runner = UnitTestRunner(node_test_suite)
elif my_test_type == 'ScalabilityTest': elif my_test_type == 'ScalabilityTest':
runner = ScalabilityTestRunner(node_test_suite) runner = ScalabilityTestRunner(self)
else: else:
log("testnode, Runner type %s not implemented.", my_test_type) log("testnode, Runner type %s not implemented.", my_test_type)
raise NotImplementedError raise NotImplementedError
...@@ -404,9 +404,11 @@ shared = true ...@@ -404,9 +404,11 @@ shared = true
node_test_suite.edit(test_result=test_result) node_test_suite.edit(test_result=test_result)
# get cluster configuration for this test suite, this is needed to # get cluster configuration for this test suite, this is needed to
# know slapos parameters to user for creating instances # know slapos parameters to user for creating instances
node_test_suite.edit(cluster_configuration=Utils.deunicodeData( log("Getting configuration from test suite " + str(node_test_suite.test_suite_title))
json.loads(self.test_suite_portal.generateConfiguration( generated_config = self.test_suite_portal.generateConfiguration(node_test_suite.test_suite_title)
node_test_suite.test_suite_title))['configuration_list'][0])) jsonData = json.loads(generated_config)
cluster_configuration = Utils.deunicodeData(jsonData['configuration_list'][0])
node_test_suite.edit(cluster_configuration=cluster_configuration)
# Now prepare the installation of SlapOS and create instance # Now prepare the installation of SlapOS and create instance
status_dict = runner.prepareSlapOSForTestSuite(node_test_suite) status_dict = runner.prepareSlapOSForTestSuite(node_test_suite)
# Give some time so computer partitions may start # Give some time so computer partitions may start
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment