Skip to content

slapos
slapos
Commit 88f0bcd5 authored by Xavier Thompson's avatar Xavier Thompson
Browse files
Options

software/theia: Simplify and improve embedded SR 

See merge request nexedi/slapos!1139
parents c9f8bad9 1b877749
Pipeline #21224 failed with stage
    in 0 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 533 additions and 342 deletions
    software/theia/buildout.hash.cfg
    View file @ 88f0bcd5
    ......@@ -15,24 +15,28 @@
    [instance-theia]
    _update_hash_filename_ = instance-theia.cfg.jinja.in
    md5sum = 7ea72f2300dd2019c2d27549e7128dec
    md5sum = 7d146ca31d7e44e909386e21e0a562ec
    [instance]
    _update_hash_filename_ = instance.cfg.in
    md5sum = af4d92a95dc25c2f64ddbd634996e46c
    md5sum = e211c439571e2900f9f35482c9638d06
    [instance-import]
    _update_hash_filename_ = instance-import.cfg.jinja.in
    md5sum = 29bc1c5b76d20fd46430dff5c72d192b
    md5sum = 23c3df4a889ebfa9f0a94e873e95ad3b
    [instance-export]
    _update_hash_filename_ = instance-export.cfg.jinja.in
    md5sum = e2630148998c3cdf6d03b5e884d7f464
    md5sum = 84ceb4c9ee0f07fce8518ef7517ce1d4
    [instance-resilient]
    _update_hash_filename_ = instance-resilient.cfg.jinja
    md5sum = ad9499e7355ded4975ad313442cecb7a
    [slapos-standalone-script]
    _update_hash_filename_ = slapos_standalone_script.py.jinja
    md5sum = ef5b73648513caf46573f3302953790f
    [theia-common]
    _update_hash_filename_ = theia_common.py
    md5sum = 6a25c6a7f1beb27232a3c9acd8a76500
    ......
    software/theia/instance-export.cfg.jinja.in
    View file @ 88f0bcd5
    ......@@ -92,14 +92,14 @@ inline =
    then
    echo "ERROR export script last ran on " $(date -r {{ repr(exitcodefile) }})
    exit 1
    elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
    elif [ "$(cat {{ repr(exitcodefile) }})" = 0 ]
    then
    echo "OK export script last ran on " $(date -r {{ repr(exitcodefile) }})
    exit 0
    else
    echo "ERROR export script failed on " $(date -r {{ repr(exitcodefile) }})
    cat {{ repr(errorfile) }}
    exit 1
    else
    echo "OK export script last ran on " $(date -r {{ repr(exitcodefile) }})
    exit 0
    fi
    {%- endraw %}
    ......
    software/theia/instance-import.cfg.jinja.in
    View file @ 88f0bcd5
    ......@@ -154,14 +154,14 @@ inline =
    then
    echo "ERROR import script last ran on " $(date -r {{ repr(exitcodefile) }})
    exit 1
    elif [ $( cat {{ repr(exitcodefile) }}) -ne 0 ]
    elif [ "$(cat {{ repr(exitcodefile) }})" = 0 ]
    then
    echo "OK import script last ran on " $(date -r {{ repr(exitcodefile) }})
    exit 0
    else
    echo "ERROR import script failed on " $(date -r {{ repr(exitcodefile) }})
    cat {{ repr(errorfile) }}
    exit 1
    else
    echo "OK import script last ran on " $(date -r {{ repr(exitcodefile) }})
    exit 0
    fi
    {%- endraw %}
    ......
    software/theia/instance-input-schema.json
    View file @ 88f0bcd5
    ......@@ -15,20 +15,14 @@
    ],
    "default": "running"
    },
    "embedded-sr": {
    "title": "Embedded Software URL",
    "description": "Optional URL of a software to be embedded",
    "type": "string"
    },
    "embedded-sr-type": {
    "title": "Embedded Software Type",
    "description": "Type of the optional embedded software",
    "type": "string"
    },
    "embedded-instance-parameters": {
    "title": "Embedded Instance Parameters",
    "description": "Parameters for the embedded instance, as a JSON dict",
    "type": "string"
    "initial-embedded-instance": {
    "title": "Initial Embedded Instance Configuration",
    "description": "One-shot optional JSON preconfiguration for an embedded instance. Only applied once when Theia is instantiated. Changing this option afterward will have no effect.",
    "type": "string",
    "examples": [
    "{\"software-url\": \"~/srv/project/slapos/software/html5as-base/software.cfg\"}",
    "{\"software-url\": \"~/srv/project/slapos/software/html5as/software.cfg\", \"software-type\": \"replicate\", \"instance-parameters\": {\"replicate-quantity\": 3}}"
    ]
    },
    "frontend-guid": {
    "title": "Frontend Instance ID",
    ......
    software/theia/instance-theia.cfg.jinja.in
    View file @ 88f0bcd5
    {% set parameter_dict = dict(default_parameter_dict, **parameter_dict) %}
    {% set additional_frontend = parameter_dict['additional-frontend-guid'] %}
    {%- set parameter_dict = dict(default_parameter_dict, **parameter_dict) %}
    {%- set additional_frontend = parameter_dict['additional-frontend-guid'] %}
    {%- set embedded_instance_config = parameter_dict['initial-embedded-instance'] %}
    [buildout]
    extends =
    ......@@ -10,7 +11,6 @@ theia-environment-parts =
    slapos-repository
    runner-link
    settings.json
    request-script-template
    theia-parts =
    frontend-reload
    ......@@ -98,6 +98,9 @@ instance-promises =
    $${slapos-standalone-listen-promise:name}
    $${slapos-standalone-ready-promise:name}
    $${slapos-autorun-promise:name}
    {% if embedded_instance_config %}
    $${embedded-instance-requested-promise:name}
    {% endif %}
    [theia-listen-promise]
    <= monitor-promise-base
    ......@@ -161,6 +164,14 @@ config-service = $${slapos-autorun:service-name}
    config-expect = $${slapos-autorun:autorun}
    config-run-directory = $${directory:runner}/var/run
    {% if embedded_instance_config %}
    [embedded-instance-requested-promise]
    <= monitor-promise-base
    promise = check_command_execute
    name = embedded-instance-requested-promise.py
    config-command = $${embedded-instance-requested-promise-script:output}
    {% endif %}
    # Remote Caddy Frontend
    # ---------------------
    ......@@ -402,7 +413,7 @@ base-url = $${theia-service:base-url}
    [theia-shell]
    recipe = slapos.recipe.template:jinja2
    output = $${directory:bin}/$${:_buildout_section_name_}
    inline
    inline =
    {% raw -%}
    #!{{ bash }}
    SHELL=$BASH
    ......@@ -441,35 +452,40 @@ command =
    # Embedded Instance
    # -----------------
    {%- set embedded_sr = parameter_dict['embedded-sr'] %}
    {%- set embedded_sr_type = parameter_dict['embedded-sr-type'] %}
    {%- set embedded_instance_parameters = parameter_dict['embedded-instance-parameters'] %}
    {%- if embedded_sr %}
    {%- if embedded_sr.startswith('~/') %}
    {%- set embedded_sr = os_module.path.join(partition_root_path, embedded_sr[2:]) %}
    {%- set embedded_sr = os_module.path.normpath(embedded_sr) %}
    {%- endif %}
    [request-embedded-instance-script]
    recipe = slapos.recipe.template
    output = $${directory:project}/request_embedded.sh
    [embedded-instance-config]
    recipe = slapos.recipe.template:jinja2
    output = $${directory:etc}/$${:_buildout_section_name_}.json
    once = $${:output}
    config = {{ dumps(embedded_instance_config) }}
    context =
    key config :config
    inline =
    #!/bin/sh
    slapos supply {{ embedded_sr }} slaprunner
    slapos request "embedded_instance" {{ embedded_sr }}
    {%- if embedded_sr_type %} --type {{ embedded_sr_type }} {%- endif %}
    {%- if embedded_instance_parameters %} --parameters-file $${embedded-instance-parameters:output}
    {%- raw %}
    {{ config or "{}"}}
    {%- endraw %}
    [embedded-instance-parameters]
    recipe = slapos.recipe.template
    output = $${directory:project}/$${:_buildout_section_name_}.json
    inline = {{ embedded_instance_parameters | indent(2) }}
    {%- endif %}
    {%- endif %}
    {%- set embedded_digest = str(embedded_sr) + str(embedded_sr_type) + str(embedded_instance_parameters) %}
    {%- set embedded_digest_hash = hashlib_module.md5(embedded_digest.encode()).hexdigest() %}
    [embedded-instance-requested-promise-script]
    recipe = slapos.recipe.template:jinja2
    output = $${directory:bin}/$${:_buildout_section_name_}
    exitcode-file = $${slapos-standalone-script:embedded-request-exitcode-file}
    context =
    key exitcodefile :exitcode-file
    {%- raw %}
    inline =
    #!/bin/sh
    if ! [ -f {{ repr(exitcodefile) }} ]
    then
    echo "ERROR embedded_instance has not been requested"
    exit 1
    elif [ "$(cat {{ repr(exitcodefile) }})" = 0 ]
    then
    echo "OK embedded_instance has been sucessfully requested"
    exit 0
    else
    echo "ERROR request of embedded_instance failed"
    exit 1
    fi
    {%- endraw %}
    # SlapOS Standalone
    ......@@ -485,10 +501,14 @@ ip = {{ ipv4_random }}
    ipv4 = {{ ipv4_random }}
    ipv6 = {{ ipv6_random }}
    port = $${slapos-standalone-port:port}
    base-directory = $${directory:runner}
    software-root = $${directory:runner}/software
    instance-root = $${directory:runner}/instance
    local-software-release-root = $${directory:home}
    slapos-bin = ${buildout:bin-directory}/slapos
    slapos-configuration = $${directory:runner}/etc/slapos.cfg
    computer-id = slaprunner
    abstract-socket-path = $${directory:home}/standalone-{{ embedded_digest_hash[:16] }}
    abstract-socket-path = $${directory:home}/standalone-ready
    [slapos-standalone-activate]
    recipe = slapos.recipe.template
    ......@@ -500,98 +520,23 @@ inline =
    echo 'Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` activated'
    [slapos-standalone-script]
    recipe = slapos.recipe.template
    recipe = slapos.recipe.template:jinja2
    output = $${directory:bin}/$${:_buildout_section_name_}
    inline =
    #!${python-for-standalone:executable}
    import glob
    import json
    import os
    import signal
    import socket
    import subprocess
    import sys
    import time
    import traceback
    import slapos.slap.standalone
    # Include this hash, so that if it changes the standalone service will be restarted
    # {{ embedded_digest_hash }}
    shared_parts = """{{ '''${buildout:shared-part-list}''' | indent(2) }}"""
    shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
    partition_forward_configuration = (
    slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
    master_url="$${slap-connection:server-url}",
    computer="$${slap-connection:computer-id}",
    partition="$${slap-connection:partition-id}",
    cert="$${slap-connection:cert-file}",
    key="$${slap-connection:key-file}",
    software_release_list=(
    'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
    ),
    ),
    )
    standalone = slapos.slap.standalone.StandaloneSlapOS(
    "$${directory:runner}",
    "$${slapos-standalone-config:ipv4}",
    $${slapos-standalone-config:port},
    computer_id="$${slapos-standalone-config:computer-id}",
    shared_part_list=shared_part_list,
    software_root="$${directory:runner}/software",
    instance_root="$${directory:runner}/instance",
    partition_forward_configuration=partition_forward_configuration,
    slapos_bin="${buildout:bin-directory}/slapos",
    local_software_release_root="$${slapos-standalone-config:local-software-release-root}",
    )
    def signal_handler(signum, frame):
    print("Signal {signum} received".format(signum=signum))
    sys.exit()
    signal.signal(signal.SIGTERM, signal_handler)
    standalone.start()
    try:
    partition_count = 20
    print("Standalone SlapOS: Formatting %d partitions" % partition_count)
    standalone.format(partition_count, '$${slapos-standalone-config:ipv4}', '$${slapos-standalone-config:ipv6}')
    print("Standalone SlapOS for computer `$${slapos-standalone-config:computer-id}` started")
    # Run instance at least once, to start the supervisor managing instances.
    try:
    standalone.waitForInstance(max_retry=0)
    except slapos.slap.standalone.SlapOSNodeCommandError as e:
    print("Error instanciating: {}".format(e))
    {%- if embedded_sr %}
    # Compatibility layer
    try:
    for cp in standalone.computer.getComputerPartitionList():
    if cp.getInstanceParameterDict().get("instance_title") == "Embedded Instance":
    print("Renaming 'Embedded Instance' into 'embedded_instance'")
    cp.rename(new_name="embedded_instance")
    break
    except Exception:
    print("Exception in compatibility layer, printing and moving on")
    traceback.print_exc()
    # Run request script
    print("Running SlapOS script $${request-embedded-instance-script:output}")
    slapos_env = {
    'PATH': os.path.dirname(standalone._slapos_bin),
    'SLAPOS_CONFIGURATION': standalone._slapos_config,
    'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config
    }
    subprocess.call(("$${request-embedded-instance-script:output}",), env=slapos_env)
    {%- endif %}
    s = socket.socket(socket.AF_UNIX)
    s.bind("\0$${slapos-standalone-config:abstract-socket-path}")
    s.listen(5)
    print("Standalone SlapOS ready")
    while True:
    s.accept()[0].close()
    finally:
    print("Stopping standalone subsystem")
    standalone.stop()
    print("Exiting")
    embedded-request-exitcode-file = $${directory:etc}/embedded-request-exitcode
    shared-part-list =
    {{ """${buildout:shared-part-list}""" | indent(2) }}
    context =
    raw python_for_standalone ${python-for-standalone:executable}
    raw request_script_path $${directory:project}/request-embedded-instance.sh
    raw parameters_file_path $${directory:project}/embedded-instance-parameters.json
    key request_script_template request-script-example:inline
    key shared_part_list :shared-part-list
    key embedded_request_exitcode_file :embedded-request-exitcode-file
    key embedded_instance_config embedded-instance-config:output
    key home_path directory:home
    section slap_connection slap-connection
    section slapos_standalone_config slapos-standalone-config
    url = ${slapos-standalone-script:output}
    [slapos-standalone]
    recipe = slapos.recipe.template
    ......@@ -725,18 +670,48 @@ recipe = slapos.cookbook:symbolic.link
    target-directory = $${directory:project}
    link-binary = $${directory:runner}
    [request-script-template]
    recipe = slapos.recipe.template
    [request-script-example]
    recipe = slapos.recipe.template:jinja2
    output = $${directory:project}/$${:_buildout_section_name_}.sh
    software_url = ~/srv/project/slapos/software/html5as-base/software.cfg
    request_options = html5as-1 $${:software_url}
    header_text =
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # This template is generated automatically by buildout. #
    # Any changes to this file may be overwritten. #
    # Copy and adapt it to create your own request script. #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    context =
    key software_url :software_url
    key request_options :request_options
    key header_text :header_text
    inline =
    #!/bin/sh
    # This template is generated automatically, copy it in order to supply and request.
    # Any manual change to this file may be lost.
    software_name=html5as-base #replace the software name writen in ~/srv/project/slapos/software/
    software_release_uri=~/srv/project/slapos/software/$software_name/software.cfg
    # slapos supply is used to add the software to the software list to be supplied to a node.
    slapos supply $software_release_uri slaprunner
    # slapos request the allocation of an instance to the master.
    # slapos request also gets status and parameters of the instance if it has any
    # (slapos request is meant to be run multiple time until you get the status).
    slapos request $software_name'_1' $software_release_uri
    {%- raw %}
    #!/bin/sh -e
    {{ header_text }}
    # slapos supply <url> <node> registers a software for installation on a node.
    #
    # A software is uniquely identified by its URL (the URL may be a local path).
    # You may choose from softwares available in ~/srv/project/slapos/software.
    #
    # The one and only SlapOS Node embedded inside Theia is called 'slaprunner'.
    #
    # For more information, run:
    # slapos help supply
    #
    slapos supply {{ software_url }} slaprunner
    # slapos request <name> <url> registers an instance for allocation on a node.
    #
    # An instance is uniquely identified by its name (and its requester).
    #
    # It will be allocated on a node where the software URL is already supplied.
    # Inside Theia that node can only be 'slaprunner'.
    #
    # For more information, run:
    # slapos help request
    #
    slapos request {{ request_options }}
    {% endraw %}
    software/theia/instance.cfg.in
    View file @ 88f0bcd5
    ......@@ -31,6 +31,7 @@ pull-backup = template-pull-backup:output
    recipe = slapos.recipe.template:jinja2
    url = ${instance-theia:output}
    output = $${buildout:directory}/instance-theia.cfg
    extensions = jinja2.ext.do
    context =
    jsonkey default_parameter_dict :default-parameters
    key parameter_dict slap-configuration:configuration
    ......@@ -38,14 +39,12 @@ context =
    key partition_root_path buildout:directory
    key ipv6_random slap-configuration:ipv6-random
    key ipv4_random slap-configuration:ipv4-random
    import os_module os
    import hashlib_module hashlib
    import os os
    import json json
    default-parameters =
    {
    "autorun": "running",
    "embedded-sr": null,
    "embedded-sr-type": null,
    "embedded-instance-parameters": null,
    "initial-embedded-instance": null,
    "frontend-name": "Theia Frontend",
    "frontend-sr": "$${:frontend-sr}",
    "frontend-sr-type": "RootSoftwareInstance",
    ......
    software/theia/slapos_standalone_script.py.jinja 0 → 100644
    View file @ 88f0bcd5
    #!{{ python_for_standalone }}
    import contextlib
    import glob
    import json
    import logging
    import os
    import re
    import signal
    import socket
    import subprocess
    import sys
    import time
    import slapos.slap.standalone
    from slapos.util import unicode2str
    logging.basicConfig(format="[%(asctime)s] %(message)s", level=logging.DEBUG)
    REQUEST_SCRIPT_RAW_TEMPLATE = """{{ request_script_template }}"""
    {% raw -%}
    REQUEST_SCRIPT_TEMPLATE = re.sub(
    r"{{\s*(\w+)\s*}}",
    r"{\1}",
    REQUEST_SCRIPT_RAW_TEMPLATE,
    )
    {%- endraw %}
    REQUEST_SCRIPT_HEADER_TEXT = """
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # This script is generated once by your Theia's SlapOS Standalone service. #
    # #
    # It was used to pre-request an instance inside Theia's embedded SlapOS #
    # according to the preconfiguration parameters given to your Theia. #
    # #
    # It will not be overwritten or recreated. #
    # It will not be launched automatically. #
    # #
    # You can edit it and run it to modify your embedded instance. #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    """.strip()
    def signal_handler(signum, frame):
    logging.info("Signal %d received", signum)
    sys.exit()
    @contextlib.contextmanager
    def setupStandalone():
    partition_forward_configuration = (
    slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
    master_url={{ repr(slap_connection['server-url']) }},
    computer={{ repr(slap_connection['computer-id']) }},
    partition={{ repr(slap_connection['partition-id']) }},
    cert={{ repr(slap_connection['cert-file']) }},
    key={{ repr(slap_connection['key-file']) }},
    software_release_list=(
    'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
    ),
    ),
    )
    shared_parts = {{ repr(shared_part_list) }}
    shared_part_list = [x.strip() for x in shared_parts.splitlines() if x.strip()]
    standalone = slapos.slap.standalone.StandaloneSlapOS(
    {{ repr(slapos_standalone_config['base-directory']) }},
    {{ repr(slapos_standalone_config['ipv4']) }},
    {{ slapos_standalone_config['port'] }},
    computer_id={{ repr(slapos_standalone_config['computer-id']) }},
    shared_part_list=shared_part_list,
    software_root={{ repr(slapos_standalone_config['software-root']) }},
    instance_root={{ repr(slapos_standalone_config['instance-root']) }},
    partition_forward_configuration=partition_forward_configuration,
    slapos_bin={{ repr(slapos_standalone_config['slapos-bin']) }},
    local_software_release_root={{ repr(slapos_standalone_config['local-software-release-root']) }},
    )
    standalone.start()
    try:
    partition_count = 20
    logging.info("Standalone SlapOS: Formatting %d partitions", partition_count)
    standalone.format(partition_count, {{ repr(slapos_standalone_config['ipv4']) }}, {{ repr(slapos_standalone_config['ipv6']) }})
    logging.info("Standalone SlapOS for computer `%s` started", {{ repr(slapos_standalone_config['computer-id']) }})
    # Run instance at least once, to start the supervisor managing instances.
    try:
    standalone.waitForInstance(max_retry=0)
    except slapos.slap.standalone.SlapOSNodeCommandError as e:
    logging.info("Error instanciating: {}".format(e))
    yield standalone
    finally:
    logging.info("Stopping standalone subsystem")
    standalone.stop()
    logging.info("Exiting")
    def parseEmbeddedInstanceConfig(config_json_file):
    with open(config_json_file) as f:
    try:
    config = json.load(f)
    except json.JSONDecodeError:
    logging.error("%s is not valid JSON", config_json_file)
    raise
    assert(isinstance(config, dict))
    if config:
    software_url = unicode2str(config['software-url'])
    software_type = config.get('software-type')
    instance_parameters = config.get('instance-parameters')
    assert(isinstance(software_url, str))
    if software_type:
    software_type = unicode2str(software_type)
    assert(isinstance(software_type, str))
    if instance_parameters:
    assert(isinstance(instance_parameters, dict))
    return software_url, software_type, instance_parameters
    def createRequestScript(software_url, software_type, instance_parameters):
    # Generate request script
    request_script_path = {{ repr(request_script_path) }}
    parameters_file_path = {{ repr(parameters_file_path) }}
    request_options = "embedded_instance " + software_url
    if software_type:
    request_options += ' --type ' + software_type
    if instance_parameters:
    with open(parameters_file_path, 'w') as f:
    json.dump(instance_parameters, f)
    request_options += ' --parameters-file ' + parameters_file_path
    with open(request_script_path, 'w') as f:
    f.write(REQUEST_SCRIPT_TEMPLATE.format(
    header_text = REQUEST_SCRIPT_HEADER_TEXT,
    software_url = software_url,
    request_options = request_options,
    ))
    f.write("\n")
    os.fchmod(f.fileno(), 0o755)
    return request_script_path
    def main():
    signal.signal(signal.SIGTERM, signal_handler)
    with setupStandalone() as standalone:
    config_json_file = {{ repr(embedded_instance_config) }}
    done_file = config_json_file + '.done'
    if not os.path.exists(done_file):
    with open(done_file, 'w'): pass
    try:
    config = parseEmbeddedInstanceConfig(config_json_file)
    except Exception:
    logging.info("Failed to parse embedded instance config", exc_info=True)
    config = None
    if config:
    try:
    request_script_path = createRequestScript(*config)
    logging.info("Calling %s", request_script_path)
    exitcode = subprocess.call((request_script_path,), env={
    'HOME': {{ repr(home_path) }},
    'PATH': os.path.dirname(standalone._slapos_bin),
    'SLAPOS_CONFIGURATION': standalone._slapos_config,
    'SLAPOS_CLIENT_CONFIGURATION': standalone._slapos_config,
    })
    with open({{ repr(embedded_request_exitcode_file) }}, 'w') as f:
    f.write(str(exitcode))
    except Exception:
    logging.info("Failed to request embedded instance", exc_info=True)
    s = socket.socket(socket.AF_UNIX)
    s.bind({{ repr('\0' + slapos_standalone_config['abstract-socket-path']) }})
    s.listen(5)
    logging.info("Standalone SlapOS ready")
    while True:
    s.accept()[0].close()
    main()
    software/theia/software.cfg
    View file @ 88f0bcd5
    ......@@ -19,17 +19,9 @@ extends =
    ./buildout.hash.cfg
    parts =
    theia-wrapper
    slapos-cookbook
    python-for-resiliency
    python-for-standalone
    instance-theia
    instance
    instance-import
    instance-export
    instance-resilient
    theia-common
    theia-export
    # default for slapos-standalone
    shared-part-list =
    ......@@ -95,6 +87,10 @@ output = ${buildout:directory}/instance-export.cfg.jinja
    [instance-resilient]
    <= download-base
    [slapos-standalone-script]
    <= download-base
    destination = ${buildout:directory}/slapos_standalone_script.py.jinja
    [theia-common]
    <= download-base
    destination = ${buildout:directory}/theia_common.py
    ......@@ -102,10 +98,12 @@ destination = ${buildout:directory}/theia_common.py
    [theia-export]
    <= download-base
    destination = ${buildout:directory}/theia_export.py
    depends = ${theia-common:recipe}
    [theia-import]
    <= download-base
    destination = ${buildout:directory}/theia_import.py
    depends = ${theia-common:recipe}
    # Utilities
    ......
    software/theia/test/project_tests.py
    View file @ 88f0bcd5
    ......@@ -62,10 +62,9 @@ class ERP5Mixin(object):
    _test_software_url = erp5_software_release_url
    _connexion_parameters_regex = re.compile(r"{.*}", re.DOTALL)
    def _getERP5ConnexionParameters(self, software_type='export'):
    slapos = self._getSlapos(software_type)
    out = subprocess.check_output(
    (slapos, 'request', 'test_instance', self._test_software_url),
    def _getERP5ConnexionParameters(self, instance_type='export'):
    out = self.captureSlapos(
    'request', 'test_instance', self._test_software_url,
    stderr=subprocess.STDOUT,
    )
    print(out)
    ......@@ -110,10 +109,10 @@ class ERP5Mixin(object):
    raise Exception("Found several partitions for ERP5 %s" % servicename)
    return found.pop()
    def _getERP5PartitionPath(self, software_type, servicename, *paths):
    def _getERP5PartitionPath(self, instance_type, servicename, *paths):
    partition = self._getERP5Partition(servicename)
    return self._getPartitionPath(
    software_type, 'srv', 'runner', 'instance', partition, *paths)
    return self.getPartitionPath(
    instance_type, 'srv', 'runner', 'instance', partition, *paths)
    class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
    ......@@ -161,16 +160,15 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
    # Update ERP5 parameters
    print('Requesting ERP5 with parameters %s' % params)
    slapos = self._getSlapos()
    subprocess.check_call((slapos, 'request', 'test_instance', self._test_software_url, '--parameters', params))
    self.checkSlapos('request', 'test_instance', self._test_software_url, '--parameters', params)
    # Process twice to propagate parameter changes
    for _ in range(2):
    subprocess.check_call((slapos, 'node', 'instance'))
    self.checkSlapos('node', 'instance')
    # Restart cron (actually all) services to let them take the new date into account
    # XXX this should not be required, updating ERP5 parameters should be enough
    subprocess.call((slapos, 'node', 'restart', 'all'))
    self.callSlapos('node', 'restart', 'all')
    # Wait until after the programmed backup date, and a bit more
    t = (soon - datetime.now()).total_seconds()
    ......@@ -213,9 +211,8 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
    self.assertNotIn(self._erp5_new_title, out)
    # Stop all services
    slapos = self._getSlapos()
    print("Stop all services")
    subprocess.call((slapos, 'node', 'stop', 'all'))
    self.callSlapos('node', 'stop', 'all')
    # Manually restore mariadb from backup
    mariadb_restore_script = os.path.join(mariadb_partition, 'bin', 'restore-from-backup')
    ......
    software/theia/test/test.py
    View file @ 88f0bcd5
    ......@@ -52,11 +52,53 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(theia_soft
    class TheiaTestCase(SlapOSInstanceTestCase):
    __partition_reference__ = 'T' # for supervisord sockets in included slapos
    @classmethod
    def getPath(cls, *components):
    return os.path.join(cls.computer_partition_root_path, *components)
    @classmethod
    def _getSlapos(cls):
    partition_root = cls.computer_partition_root_path
    slapos = os.path.join(partition_root, 'srv', 'runner', 'bin', 'slapos')
    return slapos
    try:
    return cls._theia_slapos
    except AttributeError:
    cls._theia_slapos = slapos = cls.getPath('srv', 'runner', 'bin', 'slapos')
    return slapos
    @classmethod
    def callSlapos(cls, *command, **kwargs):
    return subprocess.call((cls._getSlapos(),) + command, **kwargs)
    @classmethod
    def checkSlapos(cls, *command, **kwargs):
    return subprocess.check_call((cls._getSlapos(),) + command, **kwargs)
    @classmethod
    def captureSlapos(cls, *command, **kwargs):
    kwargs.setdefault('universal_newlines', kwargs.pop('text', None))
    return subprocess.check_output((cls._getSlapos(),) + command, **kwargs)
    @classmethod
    def requestInstance(cls, parameter_dict=None, state='started'):
    cls.slap.request(
    software_release=cls.getSoftwareURL(),
    software_type=cls.getInstanceSoftwareType(),
    partition_reference=cls.default_partition_reference,
    partition_parameter_kw=parameter_dict,
    state=state
    )
    @classmethod
    def restartService(cls, service):
    with cls.slap.instance_supervisor_rpc as supervisor:
    for process_info in supervisor.getAllProcessInfo():
    service_name = process_info['name']
    if service in service_name:
    service_id = '%s:%s' % (process_info['group'], service_name)
    supervisor.stopProcess(service_id)
    supervisor.startProcess(service_id)
    break
    else:
    raise Exception("Service %s not found" % service)
    class TestTheia(TheiaTestCase):
    ......@@ -104,7 +146,7 @@ class TestTheia(TheiaTestCase):
    # there's a public folder to serve file
    with open('{}/srv/frontend-static/public/test_file'.format(
    self.computer_partition_root_path), 'w') as f:
    self.getPath()), 'w') as f:
    f.write("hello")
    resp = self.get(urljoin(authenticated_url, '/public/'))
    self.assertIn('test_file', resp.text)
    ......@@ -128,10 +170,9 @@ class TestTheia(TheiaTestCase):
    self.assertIn('ipv6', self.connection_parameters)
    def test_theia_slapos(self):
    home = self.getPath()
    # Make sure we can use the shell and the integrated slapos command
    process = pexpect.spawnu(
    '{}/bin/theia-shell'.format(self.computer_partition_root_path),
    env={'HOME': self.computer_partition_root_path})
    process = pexpect.spawnu(home + '/bin/theia-shell', env={'HOME': home})
    # use a large enough terminal so that slapos proxy show table fit in the screen
    process.setwinsize(5000, 5000)
    ......@@ -173,10 +214,11 @@ class TestTheia(TheiaTestCase):
    process.wait()
    def test_theia_shell_execute_tasks(self):
    home = self.getPath()
    # shell needs to understand -c "command" arguments for theia tasks feature
    test_file = '{}/test file'.format(self.computer_partition_root_path)
    test_file = home + '/test file'
    subprocess.check_call([
    '{}/bin/theia-shell'.format(self.computer_partition_root_path),
    home + '/bin/theia-shell',
    '-c',
    'touch "{}"'.format(test_file)
    ])
    ......@@ -184,26 +226,23 @@ class TestTheia(TheiaTestCase):
    def test_theia_request_script(self):
    script_path = os.path.join(
    self.computer_partition_root_path,
    self.getPath(),
    'srv',
    'project',
    'request-script-template.sh',
    'request-script-example.sh',
    )
    self.assertTrue(os.path.exists(script_path))
    def test_slapos_cli(self):
    slapos = self._getSlapos()
    proxy_show_output = subprocess.check_output((slapos, 'proxy', 'show'))
    self.assertIn(b'slaprunner', proxy_show_output)
    computer_list_output = subprocess.check_output((slapos, 'computer', 'list'))
    self.assertIn(b'slaprunner', computer_list_output)
    self.assertIn(b'slaprunner', self.captureSlapos('proxy', 'show'))
    self.assertIn(b'slaprunner', self.captureSlapos('computer', 'list'))
    class TestTheiaEmbeddedSlapOSShutdown(TheiaTestCase):
    def test_stopping_instance_stops_embedded_slapos(self):
    embedded_slapos_supervisord_socket = _getSupervisordSocketPath(
    os.path.join(
    self.computer_partition_root_path,
    self.getPath(),
    'srv',
    'runner',
    'instance',
    ......@@ -231,82 +270,70 @@ class TestTheiaEmbeddedSlapOSShutdown(TheiaTestCase):
    self.assertFalse(embedded_slapos_process.is_running())
    class ReRequestMixin(object):
    def rerequest(self, parameter_dict=None, state='started'):
    software_url = self.getSoftwareURL()
    software_type = self.getInstanceSoftwareType()
    name = self.default_partition_reference
    self.slap.request(
    software_release=software_url,
    software_type=software_type,
    partition_reference=name,
    partition_parameter_kw=parameter_dict,
    state=state)
    def reinstantiate(self):
    # Process at least twice to propagate parameter changes
    try:
    self.slap.waitForInstance()
    except SlapOSNodeCommandError:
    pass
    self.slap.waitForInstance(self.instance_max_retry)
    class TestTheiaWithEmbeddedInstance(TheiaTestCase):
    sr_url = '~/bogus/sr/url.cfg'
    sr_type = 'bogus-type'
    sr_config = {"bogus": "yes"}
    regexpr = re.compile(r"([\w/\-\.]+)\s+slaprunner\s+available")
    class TestTheiaWithSR(TheiaTestCase, ReRequestMixin):
    sr_url = '~/bogus/software.cfg'
    sr_type = 'bogus_type'
    instance_parameters = '{\n"bogus_param": "bogus_value",\n"bogus_param2": "bogus_value2"\n}'
    def proxy_show(self, slapos):
    return subprocess.check_output((slapos, 'proxy', 'show'), universal_newlines=True)
    def test(self):
    slapos = self._getSlapos()
    home = self.computer_partition_root_path
    # Check that no request script was generated
    request_script = os.path.join(home, 'srv', 'project', 'request_embedded.sh')
    self.assertFalse(os.path.exists(request_script))
    # Manually request old-name 'Embedded Instance'
    old_instance_name = "Embedded Instance"
    subprocess.check_call((slapos, 'request', old_instance_name, 'bogus_url'))
    self.assertIn(old_instance_name, self.proxy_show(slapos))
    # Update Theia instance parameters
    embedded_request_parameters = {
    'embedded-sr': self.sr_url,
    'embedded-sr-type': self.sr_type,
    'embedded-instance-parameters': self.instance_parameters
    @classmethod
    def getInstanceParameterDict(cls, sr_url=None, sr_type=None, sr_config=None):
    return {
    'initial-embedded-instance': json.dumps({
    'software-url': sr_url or cls.sr_url,
    'software-type': sr_type or cls.sr_type,
    'instance-parameters': sr_config or cls.sr_config,
    }),
    }
    self.rerequest(embedded_request_parameters)
    self.reinstantiate()
    # Check that embedded instance was requested
    instance_name = "embedded_instance"
    info = self.proxy_show(slapos)
    try:
    self.assertIn(instance_name, info)
    except AssertionError:
    for filename in os.listdir(home):
    if 'standalone' in filename and '.log' in filename:
    filepath = os.path.join(home, filename)
    with open(filepath) as f:
    print("Contents of filepath: " + filepath)
    print(f.read())
    raise
    def expandUrl(self, url):
    if url.startswith('~/'):
    url = os.path.join(self.getPath(), url[2:])
    return url
    def assertSupplied(self, sr_url, info=None):
    info = info or self.captureSlapos('proxy', 'show', text=True)
    self.assertIn(sr_url, info)
    self.assertIn(sr_url, self.regexpr.findall(info))
    def assertNotSupplied(self, sr_url, info=None):
    info = info or self.captureSlapos('proxy', 'show', text=True)
    self.assertNotIn(sr_url, info)
    def assertEmbedded(self, sr_url, sr_type, config):
    proxy_info = self.captureSlapos('proxy', 'show', text=True)
    self.assertSupplied(sr_url, info=proxy_info)
    name = 'embedded_instance'
    self.assertIn(name, self.captureSlapos('service', 'list', text=True))
    info = self.captureSlapos('service', 'info', name, text=True)
    self.assertIn(sr_url, info)
    self.assertIn(sr_type, proxy_info)
    self.assertIn(repr(config).replace("u'", "'"), info)
    def assertNotEmbedded(self, sr_url, sr_type, config):
    sr_url = self.expandUrl(sr_url)
    proxy_info = self.captureSlapos('proxy', 'show', text=True)
    self.assertNotSupplied(sr_url, info=proxy_info)
    self.assertNotIn(sr_type, proxy_info)
    # Check that old-name instance was renamed
    self.assertNotIn(old_instance_name, info)
    def test(self):
    # Check that embedded instance is supplied and requested
    initial_sr_url = self.expandUrl(self.sr_url)
    self.assertEmbedded(initial_sr_url, self.sr_type, self.sr_config)
    # Check embedded instance parameters
    bogus_sr = os.path.join(home, self.sr_url[2:])
    # Change parameters for embedded instance
    sr_url = '/bogus/sr/url-2.cfg'
    sr_type = 'bogus-type-2'
    sr_config = {"bogus-2": "true"}
    self.requestInstance(
    self.getInstanceParameterDict(sr_url, sr_type, sr_config))
    self.waitForInstance()
    self.assertIsNotNone(re.search(r"%s\s+slaprunner\s+available" % (bogus_sr,), info), info)
    self.assertIsNotNone(re.search(r"%s\s+%s\s+%s" % (bogus_sr, self.sr_type, instance_name), info), info)
    # Check that parameters have not been taken into account
    self.assertNotEmbedded(sr_url, sr_type, sr_config)
    service_info = subprocess.check_output((slapos, 'service', 'info', instance_name), universal_newlines=True)
    self.assertIn("{'bogus_param': 'bogus_value', 'bogus_param2': 'bogus_value2'}", service_info)
    # Check that previous instance has not been changed
    self.assertEmbedded(initial_sr_url, self.sr_type, self.sr_config)
    class TestTheiaFrontend(TheiaTestCase):
    ......@@ -331,7 +358,9 @@ class TestTheiaEnv(TheiaTestCase):
    @classmethod
    def getInstanceParameterDict(cls):
    return {
    'embedded-sr': cls.dummy_software_path,
    'initial-embedded-instance': json.dumps({
    'software-url': cls.dummy_software_path,
    }),
    'autorun': 'stopped',
    }
    ......@@ -339,7 +368,7 @@ class TestTheiaEnv(TheiaTestCase):
    """Make sure environment variables are the same whether we use shell or supervisor services.
    """
    # The path of the env.json file expected to be generated by building the dummy software release
    env_json_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'software', 'env.json')
    env_json_path = self.getPath('srv', 'runner', 'software', 'env.json')
    # Get the pid of the theia process from the test node's instance-supervisord
    with self.slap.instance_supervisor_rpc as supervisor:
    ......@@ -357,7 +386,7 @@ class TestTheiaEnv(TheiaTestCase):
    # Start a theia shell that inherits the environment of the theia process
    # This simulates the environment of a shell launched from the browser application
    theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.computer_partition_root_path), env=theia_env)
    theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.getPath()), env=theia_env)
    try:
    theia_shell_process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
    ......@@ -377,7 +406,7 @@ class TestTheiaEnv(TheiaTestCase):
    # Note that we have two services, slapos-node-software and slapos-node-software-all
    # The later uses --all which is what we want to use here, because the software
    # is already installed and we want to install it again, this time from supervisor
    embedded_run_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'var', 'run')
    embedded_run_path = self.getPath('srv', 'runner', 'var', 'run')
    embedded_supervisord_socket_path = _getSupervisordSocketPath(embedded_run_path, self.logger)
    with getSupervisorRPC(embedded_supervisord_socket_path) as embedded_supervisor:
    previous_stop_time = embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop']
    ......@@ -410,6 +439,8 @@ class ResilientTheiaMixin(object):
    @classmethod
    def setUpClass(cls):
    super(ResilientTheiaMixin, cls).setUpClass()
    # Patch the computer root path to that of the export theia instance
    cls.computer_partition_root_path = cls.getPartitionPath('export')
    # Add resiliency files to snapshot patterns
    cls._save_instance_file_pattern_list += (
    '*/srv/export-exitcode-file',
    ......@@ -419,43 +450,53 @@ class ResilientTheiaMixin(object):
    )
    @classmethod
    def _getPartition(cls, software_type):
    def getPartitionId(cls, instance_type):
    software_url = cls.getSoftwareURL()
    for computer_partition in cls.slap.computer.getComputerPartitionList():
    partition_url = computer_partition.getSoftwareRelease()._software_release
    partition_type = computer_partition.getType()
    if partition_url == software_url and partition_type == software_type:
    return computer_partition
    raise Exception("Theia %s partition not found" % software_type)
    if partition_url == software_url and partition_type == instance_type:
    return computer_partition.getId()
    raise Exception("Theia %s partition not found" % instance_type)
    @classmethod
    def getPartitionPath(cls, instance_type='export', *paths):
    return os.path.join(cls.slap._instance_root, cls.getPartitionId(instance_type), *paths)
    @classmethod
    def _getPartitionId(cls, software_type):
    return cls._getPartition(software_type).getId()
    def _getSlapos(cls, instance_type='export'):
    return cls.getPartitionPath(instance_type, 'srv', 'runner', 'bin', 'slapos')
    @classmethod
    def _getPartitionPath(cls, software_type, *paths):
    return os.path.join(cls.slap._instance_root, cls._getPartitionId(software_type), *paths)
    def callSlapos(cls, *command, **kwargs):
    instance_type = kwargs.pop('instance_type', 'export')
    return subprocess.call((cls._getSlapos(instance_type),) + command, **kwargs)
    @classmethod
    def _getSlapos(cls, software_type='export'):
    return cls._getPartitionPath(software_type, 'srv', 'runner', 'bin', 'slapos')
    def checkSlapos(cls, *command, **kwargs):
    instance_type = kwargs.pop('instance_type', 'export')
    return subprocess.check_call((cls._getSlapos(instance_type),) + command, **kwargs)
    @classmethod
    def captureSlapos(cls, *command, **kwargs):
    kwargs.setdefault('universal_newlines', kwargs.pop('text', None))
    instance_type = kwargs.pop('instance_type', 'export')
    return subprocess.check_output((cls._getSlapos(instance_type),) + command, **kwargs)
    @classmethod
    def getInstanceSoftwareType(cls):
    return 'resilient'
    @classmethod
    def waitForInstance(cls):
    # process twice to propagate to all instances
    for _ in range(2):
    super(ResilientTheiaMixin, cls).waitForInstance()
    class TestTheiaResilientInterface(ResilientTheiaMixin, TestTheia):
    @classmethod
    def setUpClass(cls):
    super(TestTheiaResilientInterface, cls).setUpClass()
    # Patch the computer root path to that of the export theia instance
    cls.computer_partition_root_path = cls._getPartitionPath('export')
    pass
    class TestTheiaResilientWithSR(ResilientTheiaMixin, TestTheiaWithSR):
    @classmethod
    def setUpClass(cls):
    super(TestTheiaResilientWithSR, cls).setUpClass()
    # Patch the computer root path to that of the export theia instance
    cls.computer_partition_root_path = cls._getPartitionPath('export')
    class TestTheiaResilientWithEmbeddedInstance(ResilientTheiaMixin, TestTheiaWithEmbeddedInstance):
    pass
    software/theia/test/test_resiliency.py
    View file @ 88f0bcd5
    ......@@ -63,11 +63,10 @@ def setUpModule():
    class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase):
    @classmethod
    def _processEmbeddedInstance(cls, retries=0, software_type='export'):
    slapos = cls._getSlapos(software_type)
    def _processEmbeddedInstance(cls, retries=0, instance_type='export'):
    for _ in range(retries):
    try:
    output = subprocess.check_output((slapos, 'node', 'instance'), stderr=subprocess.STDOUT)
    output = cls.captureSlapos('node', 'instance', instance_type=instance_type, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError:
    continue
    print(output)
    ......@@ -77,19 +76,18 @@ class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase):
    # Sleep a bit as an attempt to workaround monitoring boostrap not being ready
    print("Wait before running slapos node instance one last time")
    time.sleep(120)
    subprocess.check_call((slapos, 'node', 'instance'))
    cls.callSlapos('node', 'instance', instance_type=instance_type)
    @classmethod
    def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, software_type='export'):
    slapos = cls._getSlapos(software_type)
    subprocess.check_call((slapos, 'supply', software_url, 'slaprunner'))
    def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, instance_type='export'):
    cls.callSlapos('supply', software_url, 'slaprunner', instance_type=instance_type)
    try:
    subprocess.check_output((slapos, 'node', 'software'), stderr=subprocess.STDOUT)
    cls.captureSlapos('node', 'software', instance_type=instance_type, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
    print(e.output)
    raise
    subprocess.check_call((slapos, 'request', instance_name, software_url))
    cls._processEmbeddedInstance(retries, software_type)
    cls.callSlapos('request', instance_name, software_url, instance_type=instance_type)
    cls._processEmbeddedInstance(retries, instance_type)
    @classmethod
    def getInstanceParameterDict(cls):
    ......@@ -136,16 +134,16 @@ class ResilienceMixin(object):
    class ExportAndImportMixin(object):
    def getExportExitfile(self):
    return self._getPartitionPath('export', 'srv', 'export-exitcode-file')
    return self.getPartitionPath('export', 'srv', 'export-exitcode-file')
    def getExportErrorfile(self):
    return self._getPartitionPath('export', 'srv', 'export-errormessage-file')
    return self.getPartitionPath('export', 'srv', 'export-errormessage-file')
    def getImportExitfile(self):
    return self._getPartitionPath('import', 'srv', 'import-exitcode-file')
    return self.getPartitionPath('import', 'srv', 'import-exitcode-file')
    def getImportErrorfile(self):
    return self._getPartitionPath('import', 'srv', 'import-errormessage-file')
    return self.getPartitionPath('import', 'srv', 'import-errormessage-file')
    def makedirs(self, path):
    try:
    ......@@ -185,7 +183,7 @@ class ExportAndImportMixin(object):
    initial_exitdate = os.path.getmtime(exitfile)
    # Call export script manually
    theia_export_script = self._getPartitionPath('export', 'bin', 'theia-export-script')
    theia_export_script = self.getPartitionPath('export', 'bin', 'theia-export-script')
    subprocess.check_call((theia_export_script,), stderr=subprocess.STDOUT)
    # Check that the export exitcode file was modified
    ......@@ -198,8 +196,8 @@ class ExportAndImportMixin(object):
    def _doTransfer(self):
    # Copy <export>/srv/backup/theia to <import>/srv/backup/theia manually
    export_backup_path = self._getPartitionPath('export', 'srv', 'backup', 'theia')
    import_backup_path = self._getPartitionPath('import', 'srv', 'backup', 'theia')
    export_backup_path = self.getPartitionPath('export', 'srv', 'backup', 'theia')
    import_backup_path = self.getPartitionPath('import', 'srv', 'backup', 'theia')
    shutil.rmtree(import_backup_path)
    shutil.copytree(export_backup_path, import_backup_path)
    ......@@ -209,7 +207,7 @@ class ExportAndImportMixin(object):
    initial_exitdate = os.path.getmtime(exitfile)
    # Call the import script manually
    theia_import_script = self._getPartitionPath('import', 'bin', 'theia-import-script')
    theia_import_script = self.getPartitionPath('import', 'bin', 'theia-import-script')
    subprocess.check_call((theia_import_script,), stderr=subprocess.STDOUT)
    # Check that the import exitcode file was updated
    ......@@ -277,11 +275,11 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
    os.remove(path)
    def customSignatureScript(self, content=None):
    custom_script = self._getPartitionPath('export', self.script_relpath)
    custom_script = self.getPartitionPath('export', self.script_relpath)
    self.customScript(custom_script, content)
    def customRestoreScript(self, content=None):
    restore_script = self._getPartitionPath('import', 'srv', 'runner-import-restore')
    restore_script = self.getPartitionPath('import', 'srv', 'runner-import-restore')
    self.customScript(restore_script, content)
    return restore_script
    ......@@ -294,7 +292,7 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
    self.customRestoreScript(content=None)
    self.cleanupExitfiles()
    try:
    os.remove(self._getPartitionPath('import', self.signature_relpath))
    os.remove(self.getPartitionPath('import', self.signature_relpath))
    except OSError:
    pass
    ......@@ -309,13 +307,13 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
    def test_custom_hash_script(self):
    errmsg = 'Bye bye'
    self.customSignatureScript(content='>&2 echo "%s"\nexit 1' % errmsg)
    custom_script = self._getPartitionPath('export', self.script_relpath)
    custom_script = self.getPartitionPath('export', self.script_relpath)
    self.assertExportFailure('Compute partitions backup signatures\n ... ERROR !',
    'Custom signature script %s failed' % os.path.abspath(custom_script),
    'and stderr:\n%s' % errmsg)
    def test_signature_mismatch(self):
    signature_file = self._getPartitionPath('import', self.signature_relpath)
    signature_file = self.getPartitionPath('import', self.signature_relpath)
    self.writeFile(signature_file, 'Bogus Hash\n', mode='a')
    self.assertImportFailure('ERROR the backup signatures do not match')
    ......@@ -344,7 +342,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
    def _prepareExport(self):
    # Copy ./resilience_dummy SR in export theia ~/srv/project/dummy
    dummy_target_path = self._getPartitionPath('export', 'srv', 'project', 'dummy')
    dummy_target_path = self.getPartitionPath('export', 'srv', 'project', 'dummy')
    shutil.copytree(os.path.dirname(dummy_software_url), dummy_target_path)
    self._test_software_url = os.path.join(dummy_target_path, 'software.cfg')
    ......@@ -352,8 +350,8 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
    self._deployEmbeddedSoftware(self._test_software_url, 'dummy_instance')
    relpath_dummy = os.path.join('srv', 'runner', 'instance', 'slappart0')
    self.export_dummy_root = dummy_root = self._getPartitionPath('export', relpath_dummy)
    self.import_dummy_root = self._getPartitionPath('import', relpath_dummy)
    self.export_dummy_root = dummy_root = self.getPartitionPath('export', relpath_dummy)
    self.import_dummy_root = self.getPartitionPath('import', relpath_dummy)
    # Check that dummy instance was properly deployed
    self.initial_log = self.checkLog(os.path.join(dummy_root, 'log.log'))
    ......@@ -373,7 +371,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
    self.assertTrue(os.path.exists(os.path.join(dummy_root, 'srv', '.backup_identity_script')))
    # Remember content of ~/etc in the import theia
    self.etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
    self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
    def _doSync(self):
    self._doExport()
    ......@@ -384,14 +382,13 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
    dummy_root = self.import_dummy_root
    # Check that the software url is correct
    adapted_test_url = self._getPartitionPath('import', 'srv', 'project', 'dummy', 'software.cfg')
    proxy_content = subprocess.check_output(
    (self._getSlapos('import'), 'proxy', 'show'), universal_newlines=True)
    adapted_test_url = self.getPartitionPath('import', 'srv', 'project', 'dummy', 'software.cfg')
    proxy_content = self.captureSlapos('proxy', 'show', instance_type='import', text=True)
    self.assertIn(adapted_test_url, proxy_content)
    self.assertNotIn(self._test_software_url, proxy_content)
    # Check that ~/etc still contains everything it did before
    etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
    etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
    self.assertTrue(set(self.etc_listdir).issubset(etc_listdir))
    # Check that ~/srv/project was exported
    ......@@ -401,7 +398,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
    self.checkLog(os.path.join(dummy_root, 'log.log'), self.initial_log, newline=None)
    # Check that ~/srv/.backup_identity_script was detected and called
    signature = self._getPartitionPath(
    signature = self.getPartitionPath(
    'import', 'srv', 'backup', 'theia', 'slappart0.backup.signature.custom')
    self.assertTrue(os.path.exists(signature))
    with open(signature) as f:
    ......@@ -418,7 +415,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
    def _doTakeover(self):
    # Start the dummy instance as a sort of fake takeover
    subprocess.check_call((self._getSlapos('import'), 'node', 'instance'))
    self.callSlapos('node', 'instance', instance_type='import')
    def _checkTakeover(self):
    # Check that dummy instance was properly re-deployed
    ......@@ -493,7 +490,7 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
    # Run two synchronisations on the same instances
    # to make sure everything still works the second time
    # Check ~/etc in import theia again
    self.etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
    self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
    self._doSync()
    self._checkSync()
    ......@@ -502,18 +499,18 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
    self._deployEmbeddedSoftware(self._test_software_url, 'test_instance', self.test_instance_max_retries)
    # Check that there is an export and import instance and get their partition IDs
    self.export_id = self._getPartitionId('export')
    self.import_id = self._getPartitionId('import')
    self.export_id = self.getPartitionId('export')
    self.import_id = self.getPartitionId('import')
    # Remember content of ~/etc in the import theia
    self.etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
    self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
    def _doSync(self):
    start = time.time()
    # Call exporter script instead of waiting for cron job
    # XXX Accelerate cron frequency instead ?
    exporter_script = self._getPartitionPath('export', 'bin', 'exporter')
    exporter_script = self.getPartitionPath('export', 'bin', 'exporter')
    transaction_id = str(int(time.time()))
    subprocess.check_call((exporter_script, '--transaction-id', transaction_id))
    ......@@ -524,7 +521,7 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
    def _checkSync(self):
    # Check that ~/etc still contains everything it did before
    etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
    etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
    self.assertTrue(set(self.etc_listdir).issubset(etc_listdir))
    def _doTakeover(self):
    ......@@ -541,9 +538,9 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
    # Check that there is an export, import and frozen instance and get their new partition IDs
    import_id = self.import_id
    export_id = self.export_id
    new_export_id = self._getPartitionId('export')
    new_import_id = self._getPartitionId('import')
    new_frozen_id = self._getPartitionId('frozen')
    new_export_id = self.getPartitionId('export')
    new_import_id = self.getPartitionId('import')
    new_frozen_id = self.getPartitionId('frozen')
    # Check that old export instance is now frozen
    self.assertEqual(export_id, new_frozen_id)
    ......
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment
    GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7