Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Joanne Hugé
slapos
Commits
bb23fc27
Commit
bb23fc27
authored
Aug 12, 2013
by
Benjamin Blanc
Committed by
Julien Muchembled
Sep 04, 2013
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add scalability test support
parent
145a2a94
Changes
11
Show whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
687 additions
and
58 deletions
+687
-58
component/cloudooo/buildout.cfg
component/cloudooo/buildout.cfg
+0
-1
setup.py
setup.py
+1
-0
slapos/recipe/erp5_bootstrap/__init__.py
slapos/recipe/erp5_bootstrap/__init__.py
+113
-7
slapos/recipe/erp5_bootstrap/template/erp5_bootstrap.in
slapos/recipe/erp5_bootstrap/template/erp5_bootstrap.in
+229
-30
slapos/recipe/scalability/__init__.py
slapos/recipe/scalability/__init__.py
+62
-0
software/erp5-util-testing/instance.cfg
software/erp5-util-testing/instance.cfg
+13
-0
software/erp5-util-testing/software.cfg
software/erp5-util-testing/software.cfg
+1
-1
stack/erp5/buildout.cfg
stack/erp5/buildout.cfg
+86
-19
stack/erp5/instance-erp5.cfg.in
stack/erp5/instance-erp5.cfg.in
+9
-0
stack/erp5/instance-scalability.cfg.in
stack/erp5/instance-scalability.cfg.in
+148
-0
stack/erp5/instance.cfg.in
stack/erp5/instance.cfg.in
+25
-0
No files found.
component/cloudooo/buildout.cfg
View file @
bb23fc27
...
@@ -17,7 +17,6 @@ command = grep parts ${buildout:develop-eggs-directory}/cloudooo.egg-link
...
@@ -17,7 +17,6 @@ command = grep parts ${buildout:develop-eggs-directory}/cloudooo.egg-link
[cloudooo-repository]
[cloudooo-repository]
recipe = slapos.recipe.build:gitclone
recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/cloudooo.git
repository = http://git.erp5.org/repos/cloudooo.git
branch = master
git-executable = ${git:location}/bin/git
git-executable = ${git:location}/bin/git
[cloudooo]
[cloudooo]
...
...
setup.py
View file @
bb23fc27
...
@@ -108,6 +108,7 @@ setup(name=name,
...
@@ -108,6 +108,7 @@ setup(name=name,
'erp5.update = slapos.recipe.erp5_update:Recipe'
,
'erp5.update = slapos.recipe.erp5_update:Recipe'
,
'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe'
,
'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe'
,
'erp5testnode = slapos.recipe.erp5testnode:Recipe'
,
'erp5testnode = slapos.recipe.erp5testnode:Recipe'
,
'scalability = slapos.recipe.scalability:Recipe'
,
'firefox = slapos.recipe.firefox:Recipe'
,
'firefox = slapos.recipe.firefox:Recipe'
,
'fontconfig = slapos.recipe.fontconfig:Recipe'
,
'fontconfig = slapos.recipe.fontconfig:Recipe'
,
'generate.mac = slapos.recipe.generatemac:Recipe'
,
'generate.mac = slapos.recipe.generatemac:Recipe'
,
...
...
slapos/recipe/erp5_bootstrap/__init__.py
View file @
bb23fc27
...
@@ -33,10 +33,61 @@ import urlparse
...
@@ -33,10 +33,61 @@ import urlparse
class
Recipe
(
GenericBaseRecipe
):
class
Recipe
(
GenericBaseRecipe
):
"""
"""
Instanciate ERP5 in Zope
Instanciate ERP5 in Zope
Input:
mysql-url
mysql url, must contain connexion informations.
zope-url (optional)
Url of zope, this url will be parsed to get these values:
protocol://username@password:hostname:port/site_id
If specified following inputs have priority on parsed zope-url values:
zope-hostname (optional)
Zope hostname.
Default value: "localhost", if not contained in zope-url.
zope-port (optional)
Zope port.
Default value: "80", if not contained in zope-url.
zope-protocol (optional)
Protocol used.
Default "http" if not contained in zope-url.
zope-password (optional)
Zope user password.
Default value: "insecure", if not contained in zope-url.
zope-username (optional)
Zope username.
Default value: "zope", if not contained in zope-url.
site-id (optional)
Site id
ex: erp5
Default value: "erp5", if not contained in zope-url.
runner-path
The path to create the runner.
scalability (optional)
Boolean, default value: "False"
If true erp5 site will fix the site consistency and
will use configurator to automaticlly install a
small and medium buisiness.
"""
"""
def
install
(
self
):
def
install
(
self
):
parsed
=
urlparse
.
urlparse
(
self
.
options
[
'mysql-url'
])
# Get mysql url
mysql_url
=
None
if
self
.
options
.
get
(
'mysql-url'
):
mysql_url
=
self
.
options
[
'mysql-url'
]
else
:
# Get url of the first database of the list
mysql_url_list
=
self
.
options
[
'mysql-url-list'
]
# Parse string to list
if
not
isinstance
(
mysql_url_list
,
list
):
mysql_url_list
=
mysql_url_list
[
1
:
len
(
mysql_url_list
)
-
1
].
split
(
','
)
mysql_url
=
mysql_url_list
[
0
]
parsed
=
urlparse
.
urlparse
(
mysql_url
)
mysql_connection_string
=
"%(database)s@%(hostname)s:%(port)s "
\
mysql_connection_string
=
"%(database)s@%(hostname)s:%(port)s "
\
"%(username)s %(password)s"
%
dict
(
"%(username)s %(password)s"
%
dict
(
database
=
parsed
.
path
.
split
(
'/'
)[
1
],
database
=
parsed
.
path
.
split
(
'/'
)[
1
],
...
@@ -46,15 +97,70 @@ class Recipe(GenericBaseRecipe):
...
@@ -46,15 +97,70 @@ class Recipe(GenericBaseRecipe):
password
=
parsed
.
password
password
=
parsed
.
password
)
)
# Init zope configuration
zope_username
=
None
zope_password
=
None
zope_hostname
=
None
zope_port
=
None
zope_protocol
=
None
# Get informations from zope url
if
self
.
options
.
get
(
'zope-url'
):
zope_parsed
=
urlparse
.
urlparse
(
self
.
options
[
'zope-url'
])
zope_parsed
=
urlparse
.
urlparse
(
self
.
options
[
'zope-url'
])
# Zope hostname
if
self
.
options
.
get
(
'zope-hostname'
):
zope_hostname
=
self
.
options
[
'zope-hostname'
]
elif
self
.
options
.
get
(
'zope-url'
):
zope_hostname
=
zope_parsed
.
hostname
else
:
zope_hostname
=
'localhost'
# Zope port
if
self
.
options
.
get
(
'zope-port'
):
zope_port
=
self
.
options
[
'zope-port'
]
elif
self
.
options
.
get
(
'zope-url'
):
zope_port
=
str
(
zope_parsed
.
port
)
else
:
zope_port
=
"8080"
# Zope username and password
if
self
.
options
.
get
(
'zope-username'
)
and
self
.
options
.
get
(
'zope-password'
):
zope_username
=
self
.
options
[
'zope-username'
]
zope_password
=
self
.
options
[
'zope-password'
]
elif
self
.
options
.
get
(
'zope-url'
):
zope_username
=
zope_parsed
.
username
zope_password
=
zope_parsed
.
password
else
:
zope_username
=
'zope'
zope_password
=
'insecure'
# Zope protocol
if
self
.
options
.
get
(
'zope-protocol'
):
zope_protocol
=
self
.
options
[
'zope-protocol'
]
elif
self
.
options
.
get
(
'zope-url'
):
zope_protocol
=
zope_parsed
.
scheme
else
:
zope_protocol
=
'http'
# Zope site-id
if
self
.
options
.
get
(
'zope-site-id'
):
zope_site_id
=
self
.
options
[
'zope-site-id'
]
elif
self
.
options
.
get
(
'zope-url'
):
zope_site_id
=
zope_parsed
.
path
.
split
(
'/'
)[
1
],
else
:
zope_site_id
=
'erp5'
# Scalability
if
self
.
options
.
get
(
'scalability'
):
scalability
=
self
.
options
[
'scalability'
]
else
:
scalability
=
"False"
config
=
dict
(
config
=
dict
(
python_path
=
sys
.
executable
,
python_path
=
sys
.
executable
,
user
=
zope_parsed
.
username
,
user
=
zope_username
,
password
=
zope_parsed
.
password
,
password
=
zope_password
,
site_id
=
zope_parsed
.
path
.
split
(
'/'
)[
1
],
site_id
=
zope_site_id
,
host
=
"%s:%s"
%
(
zope_parsed
.
hostname
,
zope_parsed
.
port
),
host
=
"%s:%s"
%
(
zope_hostname
,
str
(
zope_port
)),
protocol
=
zope_protocol
,
sql_connection_string
=
mysql_connection_string
,
sql_connection_string
=
mysql_connection_string
,
scalability
=
"True"
,
)
)
# Runners
# Runners
...
...
slapos/recipe/erp5_bootstrap/template/erp5_bootstrap.in
View file @
bb23fc27
...
@@ -3,28 +3,108 @@
...
@@ -3,28 +3,108 @@
import httplib
import httplib
import urllib
import urllib
import base64
import base64
import time
MAX_INSTALLATION_TIME = 60*30
MAX_TESTING_TIME = 60*2
MAX_GETTING_CONNECTION_TIME = 60*5
user = "%(user)s"
user = "%(user)s"
password = "%(password)s"
password = "%(password)s"
host = "%(host)s"
host = "%(host)s"
site_id = "%(site_id)s"
site_id = "%(site_id)s"
erp5_catalog_storage = 'erp5_mysql_innodb_catalog'
mysql_url = "%(sql_connection_string)s"
mysql_url = "%(sql_connection_string)s"
protocol = "%(protocol)s"
scalability = "%(scalability)s" == "True"
erp5_catalog_storage = 'erp5_mysql_innodb_catalog'
header_dict = {'Authorization': 'Basic %%s' %% \
header_dict = {'Authorization': 'Basic %%s' %% \
base64.encodestring('%%s:%%s' %% (user, password)).strip()}
base64.encodestring('%%s:%%s' %% (user, password)).strip()}
zope_connection = httplib.HTTPConnection(host)
# Check if an ERP5 site is already created, as ERP5 does support having
def getConnection():
# 2 instances in the same zope, and this script should not destroy user data
print "Getting new connection"
zope_connection.request('GET', '/isERP5SitePresent', headers=header_dict)
start_time = time.time()
result = zope_connection.getresponse()
count = 0
while MAX_GETTING_CONNECTION_TIME > time.time()-start_time:
try:
count = count + 1
if protocol == 'https':
return httplib.HTTPSConnection(host)
elif protocol == 'http':
return httplib.HTTPConnection(host)
else:
raise ValueError("Protocol not implemented")
except:
print "Getting new connection failed, retry"
time.sleep(10)
raise ValueError("Cannot get new connection after %%d try (for %%s s)" %%(count, str(time.time()-start_time)))
if result.status == 204: # and (result.read() == "False"):
def testIfExist(page, unexcepted_content="Site Error"):
print "Test if %%s exists" %%(page)
start_time = time.time()
count = 0
while MAX_TESTING_TIME > time.time()-start_time:
try:
count = count + 1
zope_connection = getConnection()
zope_connection.request('GET', '/%%s/%%s' %%(site_id, page))
result = zope_connection.getresponse()
return not unexcepted_content in result.read()
except:
print "Test if exists failed, retry..."
time.sleep(10)
raise ValueError("Cannot testIfExist after %%d try (for %%s s)" %%(count, str(time.time()-start_time)))
# Use a new connection
zope_connection = httplib.HTTPConnection(host)
def waitFor0PendingActivities():
# TODO: tolerate 1 pending activities ? (mail server...)
print "waitFor0PendingActivities.."
start_time = time.time()
count = 0
ok = False
while MAX_INSTALLATION_TIME > time.time()-start_time and not ok:
try:
count = count + 1
zope_connection = getConnection()
zope_connection.request(
'GET', '/%%s/portal_activities/getMessageList' %%(site_id),
headers=header_dict
)
result = zope_connection.getresponse()
message_list_text = result.read()
message_list = [s.strip() for s in message_list_text[1:-1].split(',')]
if len(message_list)==0:
print "There is no pending activities."
ok = True
#Hack to do not take into account persistent Alarm_installMailServer acitivities
if len(message_list)==1:
print "1 pending activity for 'Alarm_installMailServer'."
print "ok."
ok = True
print "There is %%d pending activities" %%len(message_list)
time.sleep(5)
except:
time.sleep(5)
print "Getting activities failed, retry."
if not ok:
raise ValueError("Pending activities always here after %%d check/try (for %%s s)" %%(count, str(time.time()-start_time)))
def createERP5Site():
# Check if an ERP5 site is already created, as ERP5 does support having
# 2 instances in the same zope, and this script should not destroy user data
zope_connection = getConnection()
zope_connection.request('GET', '/isERP5SitePresent', headers=header_dict)
result = zope_connection.getresponse()
if result.status == 204: # and (result.read() == "False"):
# Use a new connection
zope_connection = getConnection()
# Create the expected ERP5 instance
# Create the expected ERP5 instance
zope_connection.request(
zope_connection.request(
'POST', '/manage_addProduct/ERP5/manage_addERP5Site',
'POST', '/manage_addProduct/ERP5/manage_addERP5Site',
...
@@ -40,6 +120,125 @@ if result.status == 204: # and (result.read() == "False"):
...
@@ -40,6 +120,125 @@ if result.status == 204: # and (result.read() == "False"):
result = zope_connection.getresponse()
result = zope_connection.getresponse()
# Read result make sure the site really finished to
# Read result make sure the site really finished to
#
created the ERP5 site.
#
created the ERP5 site.
result.read()
print
result.read()
print "ERP5 site created."
print "ERP5 site created."
createERP5Site()
# Scalability case
if scalability:
while True:
count = 0
createERP5Site()
# Fix site consistency
if not testIfExist("/%%s/portal_configurator/" %%site_id):
count = count + 1
print "Going to fix site consistency..."
zope_connection = getConnection()
zope_connection.request(
'GET', '/%%s/ERP5Site_launchFixConfigurationConsistency' %%(site_id),
headers=header_dict
)
result = zope_connection.getresponse()
print "Check consistency..."
print result.read()
# Wait activities
waitFor0PendingActivities()
print "Site consistency checking: done."
# Install testing scalability business configuration if not exists
if testIfExist("/%%s/portal_configurator/" %%site_id) \
and not testIfExist("/%%s/sale_order_module/" %%site_id):
count = count + 1
print "Going to install testing scalability business configuration.."
zope_connection = getConnection()
zope_connection.request(
'GET', '/%%s/business_configuration_module/1/build' %%site_id,
headers=header_dict
)
result = zope_connection.getresponse()
print "Build scalability business configuration..."
print result.read()
# Wait activities
waitFor0PendingActivities()
print "Scalability business configuration building: done."
# Force to use random id generator
print "Force to use random id.."
try:
zope_connection = getConnection()
zope_connection.request(
'GET', '/%%s/use_random_id_scalability' %%site_id,
headers=header_dict
)
result = zope_connection.getresponse()
assert(result.read()=="1")
# Wait activities
waitFor0PendingActivities()
print "use_random_id_scalability: done."
except:
print "use_random_id_scalability: fail."
# Create scalability users
if testIfExist("/%%s/person_module/scalability_user/getTitle" %%site_id) \
and not testIfExist("/%%s/person_module/scalability_user_0/getTitle" %%site_id):
count = count + 1
print "Going to create scalability users.."
zope_connection = getConnection()
zope_connection.request(
'GET', '/%%s/init_scalability' %%site_id,
headers=header_dict
)
result = zope_connection.getresponse()
print "Creating scalaiblity users..."
assert(result.read()=="1")
# Wait activities
waitFor0PendingActivities()
print "Scalability users creation: done."
# Try to Updates roles and sleep
if count == 0:
print "Trying to update roles.."
try:
zope_connection = getConnection()
zope_connection.request(
'GET', '/%%s/update_roles_scalability' %%site_id,
headers=header_dict
)
result = zope_connection.getresponse()
assert(result.read()=="1")
# Wait activities
waitFor0PendingActivities()
print "Update roles: done."
except:
print "Update roles: fail. (but may be already done before)"
time.sleep(30)
# XXX: Hack to perform load balacing
# TODO: Use an other way to do the load balancing (for example
# using zope.conf, and using family-zope group names).
# Apply load balacing
if testIfExist("/%%s/portal_activities/manageLoadBalancing" %%site_id) \
and testIfExist("/%%s/person_module/scalability_user_0/getTitle" %%site_id):
print "Load balacing.."
try:
zope_connection = getConnection()
zope_connection.request(
'GET', '/%%s/apply_load_balancing_scalability' %%site_id,
headers=header_dict
)
result = zope_connection.getresponse()
assert("Processing Nodes" in result.read())
# Wait activities
waitFor0PendingActivities()
print "Load balancing init: done."
except:
print "Load balancing init: fail."
# sleep
time.sleep(30)
\ No newline at end of file
slapos/recipe/scalability/__init__.py
0 → 100644
View file @
bb23fc27
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import
os
from
slapos.recipe.librecipe
import
GenericBaseRecipe
class
Recipe
(
GenericBaseRecipe
):
def
install
(
self
):
path_list
=
[]
configuration_file
=
self
.
createFile
(
self
.
options
[
'configuration-file'
],
''
,
)
path_list
.
append
(
configuration_file
)
path_list
.
append
(
self
.
createPythonScript
(
self
.
options
[
'wrapper'
],
'slapos.recipe.librecipe.execute.executee'
,
[
# Executable
[
self
.
options
[
'exec-path'
],
'--erp5-url'
,
self
.
options
[
'erp5-url'
],
'--test-result-path'
,
self
.
options
[
'test-result-path'
],
'--test-suite-master-url'
,
self
.
options
[
'test-suite-master-url'
],
'--node-title'
,
self
.
options
[
'node-title'
],
'--test-suite'
,
self
.
options
[
'test-suite'
],
'--revision'
,
self
.
options
[
'revision'
],
'--log-path'
,
self
.
options
[
'log-path'
],
'--runner-path'
,
self
.
options
[
'runner-path'
],
'--erp5-location'
,
self
.
options
[
'erp5-location'
],
],
# Environment
{
'GIT_SSL_NO_VERIFY'
:
'1'
,
}
],
)
)
return
path_list
software/erp5-util-testing/instance.cfg
View file @
bb23fc27
[buildout]
[buildout]
parts =
parts =
scalability-runner
test-runner
test-runner
sh-environment
sh-environment
...
@@ -24,6 +25,18 @@ git-executable = ${git:location}/bin/git
...
@@ -24,6 +25,18 @@ git-executable = ${git:location}/bin/git
<= download-source
<= download-source
repository = ${erp5-util-repository:location}
repository = ${erp5-util-repository:location}
[scalability-runner]
recipe = slapos.cookbook:egg_test
run-scalability-test-suite = $${create-directory:bin}/runScalabilityTestSuite
run-scalability-test-suite-binary = $${buildout:bin-directory}/runScalabilityTestSuite
# The list of executables should be defined here and a combination
# of tests should dynamically generated.
#python-list = $${}
test-list =
$${erp5-util:location}
prepend-path = ${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin
environment = environment
[test-runner]
[test-runner]
recipe = slapos.cookbook:egg_test
recipe = slapos.cookbook:egg_test
run-test-suite = $${create-directory:bin}/runTestSuite
run-test-suite = $${create-directory:bin}/runTestSuite
...
...
software/erp5-util-testing/software.cfg
View file @
bb23fc27
...
@@ -45,7 +45,7 @@ branch = master
...
@@ -45,7 +45,7 @@ branch = master
[template]
[template]
recipe = slapos.recipe.template
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
url = ${:_profile_base_location_}/instance.cfg
md5sum =
6b919dd280d6972afe0f50d389ba5fe6
md5sum =
4449ebd6037fd248dd70a0ff7a6d6805
output = ${buildout:directory}/template.cfg
output = ${buildout:directory}/template.cfg
mode = 640
mode = 640
...
...
stack/erp5/buildout.cfg
View file @
bb23fc27
...
@@ -3,6 +3,7 @@
...
@@ -3,6 +3,7 @@
develop =
develop =
${:parts-directory}/slapos.cookbook-repository
${:parts-directory}/slapos.cookbook-repository
${:parts-directory}/cloudooo-repository
${:parts-directory}/cloudooo-repository
${:parts-directory}/erp5
extensions =
extensions =
slapos.zcbworkarounds
slapos.zcbworkarounds
...
@@ -14,6 +15,8 @@ find-links =
...
@@ -14,6 +15,8 @@ find-links =
http://dist.repoze.org
http://dist.repoze.org
http://www.nexedi.org/static/packages/source/
http://www.nexedi.org/static/packages/source/
http://www.owlfish.com/software/wsgiutils/download.html
http://www.owlfish.com/software/wsgiutils/download.html
https://pypi.python.org/packages/source/z/z3c.etestbrowser/
https://pypi.python.org/packages/source/z/zope.testbrowser/
allow-hosts += pybrary.net
allow-hosts += pybrary.net
...
@@ -104,6 +107,8 @@ parts =
...
@@ -104,6 +107,8 @@ parts =
eggs
eggs
testrunner
testrunner
test_suite_runner
test_suite_runner
scalability_test_suite_runner
performance_tester
# basic Xorg
# basic Xorg
libXdmcp
libXdmcp
...
@@ -133,6 +138,9 @@ parts =
...
@@ -133,6 +138,9 @@ parts =
# Local development
# Local development
slapos.cookbook-repository
slapos.cookbook-repository
erp5.util
slapos.cookbook
erp5.util-check
check-recipe
check-recipe
# Create instance template
# Create instance template
template
template
...
@@ -142,12 +150,33 @@ recipe = slapos.recipe.build:download
...
@@ -142,12 +150,33 @@ recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:filename}
url = ${:_profile_base_location_}/${:filename}
mode = 644
mode = 644
[erp5.util-check]
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command = grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
[erp5.util]
recipe = zc.recipe.egg
eggs = erp5.util
scripts =
ugly-depend-on = ${erp5:repository}
# Local development
# Local development
[slapos.cookbook-repository]
[slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone
recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.git
repository = http://git.erp5.org/repos/slapos.git
branch = erp5-cluster
#
branch = erp5-cluster
git-executable = ${git:location}/bin/git
git-executable = ${git:location}/bin/git
#repository = http://192.168.242.77:1236/slapos.git
branch = erp5-cluster-scalability
[slapos.cookbook]
recipe = zc.recipe.egg
eggs = slapos.cookbook
scripts =
ugly-depend-on = ${slapos.cookbook-repository:repository} ${slapos.cookbook-repository:branch}
[check-recipe]
[check-recipe]
recipe = plone.recipe.command
recipe = plone.recipe.command
...
@@ -166,6 +195,8 @@ context =
...
@@ -166,6 +195,8 @@ context =
key bin_directory buildout:bin-directory
key bin_directory buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key eggs_directory buildout:eggs-directory
key erp5_location erp5:location
key erp5_dev_location erp5:location
${:extra-context}
${:extra-context}
[template-mariadb]
[template-mariadb]
...
@@ -213,7 +244,7 @@ md5sum = 564006953b7d7a12d40a14b6648b32f0
...
@@ -213,7 +244,7 @@ md5sum = 564006953b7d7a12d40a14b6648b32f0
# XXX: "template.cfg" is hardcoded in instanciation recipe
# XXX: "template.cfg" is hardcoded in instanciation recipe
filename = template.cfg
filename = template.cfg
template = ${:_profile_base_location_}/instance.cfg.in
template = ${:_profile_base_location_}/instance.cfg.in
md5sum =
f71e14621d90903eba1b943401548b8a
md5sum =
3d5a16f831a3b73f9c0e2aae7abbd811
extra-context =
extra-context =
key mariadb_link_binary template-mariadb:link-binary
key mariadb_link_binary template-mariadb:link-binary
key zope_link_binary template-zope:link-binary
key zope_link_binary template-zope:link-binary
...
@@ -224,7 +255,6 @@ extra-context =
...
@@ -224,7 +255,6 @@ extra-context =
key curl_location curl:location
key curl_location curl:location
key dash_location dash:location
key dash_location dash:location
key dcron_location dcron:location
key dcron_location dcron:location
key erp5_location erp5:location
key file_location file:location
key file_location file:location
key findutils_location findutils:location
key findutils_location findutils:location
key fontconfig_location fontconfig:location
key fontconfig_location fontconfig:location
...
@@ -264,13 +294,14 @@ extra-context =
...
@@ -264,13 +294,14 @@ extra-context =
key template_zeo template-zeo:target
key template_zeo template-zeo:target
key template_zope template-zope:target
key template_zope template-zope:target
key template_zope_conf template-zope-conf:target
key template_zope_conf template-zope-conf:target
key template_scalability template-scalability:target
key wget_location wget:location
key wget_location wget:location
key zlib_location zlib:location
key zlib_location zlib:location
[template-erp5]
[template-erp5]
< = download-base
< = download-base
filename = instance-erp5.cfg.in
filename = instance-erp5.cfg.in
md5sum =
5b11875e6beba48db7d45c5d462a6d2d
md5sum =
0620af0820d202dedcf73334365a44bb
[template-neo]
[template-neo]
< = download-base
< = download-base
...
@@ -328,6 +359,11 @@ extra-context =
...
@@ -328,6 +359,11 @@ extra-context =
key gzip_location gzip:location
key gzip_location gzip:location
key logrotate_location logrotate:location
key logrotate_location logrotate:location
[template-scalability]
< = download-base
filename = instance-scalability.cfg.in
md5sum = d45b0b8e2fdb7fc11a5f5e4720a66879
[bt5-repository]
[bt5-repository]
# Format:
# Format:
# <url or path> [...]
# <url or path> [...]
...
@@ -355,7 +391,8 @@ repository_id_list = erp5
...
@@ -355,7 +391,8 @@ repository_id_list = erp5
[erp5]
[erp5]
recipe = slapos.recipe.build:gitclone
recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/erp5.git
repository = http://git.erp5.org/repos/erp5.git
branch = master
#repository = http://192.168.242.77:1235/erp5.git
#branch = master
git-executable = ${git:location}/bin/git
git-executable = ${git:location}/bin/git
[fix-products-paths]
[fix-products-paths]
...
@@ -404,16 +441,7 @@ initialization =
...
@@ -404,16 +441,7 @@ initialization =
os.environ['PATH'] = '${jsl:location}/bin:%s' % os.environ['PATH']
os.environ['PATH'] = '${jsl:location}/bin:%s' % os.environ['PATH']
os.environ['CGI_PATH'] = '${w3-validator:location}/httpd/cgi-bin'
os.environ['CGI_PATH'] = '${w3-validator:location}/httpd/cgi-bin'
[test_suite_runner]
[erp5-bin-base]
# XXX: Workaround for fact ERP5Type is not an distribution and does not
# expose entry point for test runner
recipe = zc.recipe.egg
python = python2.7
eggs = ${eggs:eggs}
extra-paths = ${eggs:extra-paths}
entry-points =
runTestSuite=Products.ERP5Type.tests.runTestSuite:main
scripts = runTestSuite
initialization =
initialization =
import os
import os
import sys
import sys
...
@@ -425,6 +453,36 @@ initialization =
...
@@ -425,6 +453,36 @@ initialization =
repository_id_list = list(reversed('''${erp5_repository_list:repository_id_list}'''.split()))
repository_id_list = list(reversed('''${erp5_repository_list:repository_id_list}'''.split()))
sys.path[0:0] = ['/'.join(['''${buildout:parts-directory}''', x]) for x in repository_id_list]
sys.path[0:0] = ['/'.join(['''${buildout:parts-directory}''', x]) for x in repository_id_list]
[test_suite_runner]
< = erp5-bin-base
# XXX: Workaround for fact ERP5Type is not an distribution and does not
# expose entry point for test runner
recipe = zc.recipe.egg
python = python2.7
eggs = ${eggs:eggs}
extra-paths = ${eggs:extra-paths}
entry-points =
runTestSuite=Products.ERP5Type.tests.runTestSuite:main
scripts = runTestSuite
[scalability_test_suite_runner]
< = erp5-bin-base
recipe = zc.recipe.egg
eggs = ${eggs:eggs}
extra-paths = ${eggs:extra-paths}
entry-points =
runScalabilityTestSuite=erp5.util.scalability.runScalabilityTestSuite:main
scripts = runScalabilityTestSuite
[performance_tester]
< = erp5-bin-base
recipe = zc.recipe.egg
eggs = ${eggs:eggs}
extra-paths = ${eggs:extra-paths}
entry-points =
performance_tester=erp5.util.benchmark.performance_tester:main
scripts = performance_tester
[eggs]
[eggs]
recipe = zc.recipe.egg
recipe = zc.recipe.egg
python = python2.7
python = python2.7
...
@@ -465,7 +523,8 @@ eggs =
...
@@ -465,7 +523,8 @@ eggs =
feedparser
feedparser
argparse
argparse
validictory
validictory
erp5.util
# erp5.util doesn't have to be here because is it developed (at part/erp5)
# erp5.util
huBarcode
huBarcode
qrcode
qrcode
spyne
spyne
...
@@ -509,7 +568,12 @@ eggs =
...
@@ -509,7 +568,12 @@ eggs =
# Needed for parsing .po files from our Localizer subset
# Needed for parsing .po files from our Localizer subset
polib
polib
# performance_tester dependencies
z3c.etestbrowser
zope.testbrowser
# runScalabilityTestSuite dependencie
slapos.core
psutil
# parameterizing the version of the generated python interpreter name by the
# parameterizing the version of the generated python interpreter name by the
# python section version causes dependency between this egg section and the
# python section version causes dependency between this egg section and the
# installation of python, which we don't want on an instance
# installation of python, which we don't want on an instance
...
@@ -536,7 +600,6 @@ entry-points =
...
@@ -536,7 +600,6 @@ entry-points =
scripts = zodbanalyze
scripts = zodbanalyze
[cloudooo-repository]
[cloudooo-repository]
branch = master
revision = 5c67568c403239bd8e25993602d03c553236fcec
revision = 5c67568c403239bd8e25993602d03c553236fcec
[mysql-python]
[mysql-python]
...
@@ -568,6 +631,10 @@ scripts =
...
@@ -568,6 +631,10 @@ scripts =
zodbpack
zodbpack
[versions]
[versions]
z3c.etestbrowser = 2.0.0
zope.testbrowser =
slapos.core = 0.32.3
psutil =
# pin Acquisition and Products.DCWorkflow to Nexedi flavour of eggs
# pin Acquisition and Products.DCWorkflow to Nexedi flavour of eggs
Acquisition = 2.13.7nxd001
Acquisition = 2.13.7nxd001
Products.DCWorkflow = 2.2.3nxd002
Products.DCWorkflow = 2.2.3nxd002
...
@@ -658,7 +725,7 @@ coverage = 3.6
...
@@ -658,7 +725,7 @@ coverage = 3.6
csp-eventlet = 0.7.0
csp-eventlet = 0.7.0
elementtree = 1.2.7-20070827-preview
elementtree = 1.2.7-20070827-preview
erp5.recipe.cmmiforcei686 = 0.1.3
erp5.recipe.cmmiforcei686 = 0.1.3
erp5.util =
0.4.34
erp5.util =
erp5diff = 0.8.1.5
erp5diff = 0.8.1.5
eventlet = 0.12.1
eventlet = 0.12.1
feedparser = 5.1.3
feedparser = 5.1.3
...
...
stack/erp5/instance-erp5.cfg.in
View file @
bb23fc27
...
@@ -3,6 +3,14 @@
...
@@ -3,6 +3,14 @@
{% set frontend_dict = slapparameter_dict.get('frontend', {}) %}
{% set frontend_dict = slapparameter_dict.get('frontend', {}) %}
{% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
{% set has_frontend = frontend_dict.get('software-url', '') != '' -%}
{% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}
{% set site_id = slapparameter_dict.get('site-id', 'erp5') -%}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[request-common]
[request-common]
recipe = slapos.cookbook:request.serialised
recipe = slapos.cookbook:request.serialised
software-url = ${slap-connection:software-release-url}
software-url = ${slap-connection:software-release-url}
...
@@ -17,6 +25,7 @@ config =
...
@@ -17,6 +25,7 @@ config =
${:extra-config}
${:extra-config}
extra-config =
extra-config =
config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
state = ${slap-configuration:instance-state}
{% macro request(name, software_type, config_key, config={}, ret={'url': True}) -%}
{% macro request(name, software_type, config_key, config={}, ret={'url': True}) -%}
{% do config.update(slapparameter_dict.get(config_key, {})) -%}
{% do config.update(slapparameter_dict.get(config_key, {})) -%}
...
...
stack/erp5/instance-scalability.cfg.in
0 → 100644
View file @
bb23fc27
{% if slap_software_type == software_type -%}
{% set site_id = slapparameter_dict.get('cluster').get('site-id', 'erp5') -%}
{% set promise_path = slapparameter_dict.get('promise-path', 'erp5') -%}
[buildout]
extends =
parts =
erp5-cluster
binary-wrap-launcher
binary-wrap-performance-tester
erp5-bootstrap
erp5-promise
promise-erp5-site
promise-scalability-user-data
scalability-launcher-instance
publish
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
# Request erp5-cluster instance
[erp5-cluster]
recipe = slapos.cookbook:request.serialised
sla = computer_guid
server-url = ${slap-connection:server-url}
key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
name = ERP5 Cluster
software-url = ${slap-connection:software-release-url}
software-type = default
sla-computer_guid = {{ slapparameter_dict.get('launcher-computer-guid', computer_id) }}
{# Add a zope scalability family to the erp5 cluster configuration, -#}
{# this zope family is used by erp5-bootstrap and may serve to scalability monitoring tools. -#}
{% set additional_scalability_zope = {'scalability':{'family':'scalability', 'computer-guid':computer_id}} -%}
{% do slapparameter_dict.get('cluster').get('zope-partition-dict').update( additional_scalability_zope ) -%}
{% set cluster_parameter_dict = slapparameter_dict.get('cluster', {}) -%}
config = {{ cluster_parameter_dict.keys() | join(' ') }}
{% for key, value in cluster_parameter_dict.items() -%}
config-{{ key }} = {{ dumps(value) }}
{% endfor -%}
return = mariadb-database-list family-scalability family-user memcached-persistent-url memcached-volatile-url cloudooo-url
state = ${slap-configuration:instance-state}
# Create wrapper
[binary-wrap-launcher]
recipe = slapos.cookbook:wrapper
log-path = ${basedirectory:log}
binary-path = {{ bin_directory }}/runScalabilityTestSuite
wrapper-path = ${rootdirectory:bin}/runScalabilityTestSuite
output = ${binary-wrap-launcher:binary-path}
site-id = "/{{ site_id }}"
# Suppose that there is a user zope family in the configuration
erp5-url = ${erp5-cluster:connection-family-user}${:site-id}
parameters-extra = $*
command-line = ${binary-wrap-launcher:binary-path}
return = url
# Create wrapper
[binary-wrap-performance-tester]
recipe = slapos.cookbook:wrapper
binary-path = {{ bin_directory }}/performance_tester
wrapper-path = ${rootdirectory:bin}/performance_tester
output = ${binary-wrap-performance-tester:binary-path}
parameters-extra = $*
command-line = ${binary-wrap-performance-tester:binary-path}
return = url
[scalability-launcher-instance]
recipe = slapos.cookbook:scalability
configuration-file = ${rootdirectory:etc}/scalability_runner.conf
wrapper = ${basedirectory:services}/scalability_runner
exec-path = ${binary-wrap-launcher:wrapper-path}
erp5-url = ${binary-wrap-launcher:erp5-url}
test-result-path = {{ slapparameter_dict.get('test-result-path') }}
revision = {{ slapparameter_dict.get('test-suite-revision') }}
node-title = {{ slapparameter_dict.get('node-title', 'Unknown') }}
test-suite = {{ slapparameter_dict.get('test-suite') }}
test-suite-master-url = {{ slapparameter_dict.get('test-suite-master-url') }}
log-path = ${binary-wrap-launcher:log-path}
runner-path = ${binary-wrap-performance-tester:wrapper-path}
erp5-location = {{ erp5_location }}
# Create partition's directories
[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = ${rootdirectory:var}/log
services = ${rootdirectory:etc}/run
promises = ${rootdirectory:etc}/promise/
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
var = ${buildout:directory}/var
bin = ${buildout:directory}/bin
[erp5-bootstrap]
recipe = slapos.cookbook:erp5.bootstrap
runner-path = ${basedirectory:services}/erp5-bootstrap
#mysql-url = ${erp5-cluster:mariadb-url}
mysql-url-list = ${erp5-cluster:connection-mariadb-database-list}
zope-username = zope
zope-password = insecure
zope-site-id = {{ site_id }}
zope-url = ${erp5-cluster:connection-family-scalability}
scalability = True
[erp5-promise]
recipe = slapos.cookbook:erp5.promise
promise-path = {{ promise_path }}
kumofs-url = ${erp5-cluster:connection-memcached-persistent-url}
memcached-url = ${erp5-cluster:connection-memcached-volatile-url}
cloudooo-url = ${erp5-cluster:connection-cloudooo-url}
smtp-url =
bt5 =
bt5-repository-url =
[promise-erp5-site]
recipe = slapos.cookbook:check_url_available
path = ${basedirectory:promises}/erp5site
site-id = "/{{ site_id }}"
url = ${erp5-cluster:connection-family-scalability}${:site-id}
dash_path = {{ dash_location }}/bin/dash
curl_path = {{ curl_location }}/bin/curl
[promise-scalability-user-data]
recipe = slapos.cookbook:check_url_available
path = ${basedirectory:promises}/scalabilityData
site-id = "/{{ site_id }}"
page = "/person_module/scalability_user_0/getTitle"
url = ${erp5-cluster:connection-family-scalability}${:site-id}${:page}
dash_path = {{ dash_location }}/bin/dash
curl_path = {{ curl_location }}/bin/curl
[publish]
recipe = slapos.cookbook:publish.serialised
erp5-frontend-url = ${scalability-launcher-instance:erp5-url}
erp5-scalability-url = ${erp5-cluster:connection-family-scalability}
{% endif %}
stack/erp5/instance.cfg.in
View file @
bb23fc27
...
@@ -187,6 +187,29 @@ extra-context =
...
@@ -187,6 +187,29 @@ extra-context =
# Must match the key id in [switch-softwaretype] which uses this section.
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type mariadb
raw software_type mariadb
[dynamic-template-scalability-parameters]
erp5 = {{ erp5_location }}
erp5-dev = {{ erp5_dev_location }}
local-bt5-repository = {{ local_bt5_repository }}
curl-location = {{ curl_location }}
dash-location = {{ dash_location }}
bin-directory = {{ bin_directory }}
[dynamic-template-scalability]
< = jinja2-template-base
template = {{ template_scalability }}
filename = instance-scalability.cfg
extensions = jinja2.ext.do
extra-context =
key bin_directory dynamic-template-scalability-parameters:bin-directory
key curl_location dynamic-template-scalability-parameters:curl-location
key dash_location dynamic-template-scalability-parameters:dash-location
key erp5_location dynamic-template-scalability-parameters:erp5-dev
key local_bt5_repository dynamic-template-scalability-parameters:local-bt5-repository
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type test
[switch-softwaretype]
[switch-softwaretype]
recipe = slapos.recipe.build
recipe = slapos.recipe.build
script =
script =
...
@@ -214,6 +237,7 @@ slapos_update_promise = ${:slapos_promise}
...
@@ -214,6 +237,7 @@ slapos_update_promise = ${:slapos_promise}
override = {{ dumps(override_switch_softwaretype |default) }}
override = {{ dumps(override_switch_softwaretype |default) }}
default = ${dynamic-template-erp5:rendered}
default = ${dynamic-template-erp5:rendered}
test = ${dynamic-template-scalability:rendered}
kumofs = ${dynamic-template-kumofs:rendered}
kumofs = ${dynamic-template-kumofs:rendered}
cloudooo = ${dynamic-template-cloudooo:rendered}
cloudooo = ${dynamic-template-cloudooo:rendered}
mariadb = ${dynamic-template-mariadb:rendered}
mariadb = ${dynamic-template-mariadb:rendered}
...
@@ -221,3 +245,4 @@ balancer = ${dynamic-template-balancer:rendered}
...
@@ -221,3 +245,4 @@ balancer = ${dynamic-template-balancer:rendered}
zodb-neo = ${dynamic-template-neo:rendered}
zodb-neo = ${dynamic-template-neo:rendered}
zodb-zeo = ${dynamic-template-zeo:rendered}
zodb-zeo = ${dynamic-template-zeo:rendered}
zope = ${dynamic-template-zope:rendered}
zope = ${dynamic-template-zope:rendered}
RootSoftwareInstance = ${dynamic-template-erp5-RootSoftwareInstance:rendered}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment