Commit 63078069 authored by Xavier Thompson's avatar Xavier Thompson

software/theia: Wait for proxy restart in import

Resiliency import script had a race condition where the proxy might not
have finished restarting before formatting the partitions, causing the
import script to randomly fail during format because the proxy is down
when format tries to send it updated computer information.

See merge request !1406
parent ef076948
Pipeline #28876 failed with stage
......@@ -47,7 +47,7 @@ md5sum = e2f6c483cce09f87ab1e63ae8be0daf4
[theia-import]
_update_hash_filename_ = theia_import.py
md5sum = 1a668d6203d42b4d46d56e24c7606cb2
md5sum = 45e757f216374d22f0a92d5334dc00f0
[slapos.css.in]
_update_hash_filename_ = slapos.css.in
......
......@@ -125,6 +125,7 @@ eggs =
${slapos-toolbox:eggs}
six
zc.buildout
requests
[python-for-standalone]
<= python-with-eggs
......
......@@ -4,8 +4,12 @@ import itertools
import os
import sys
import subprocess as sp
import time
import traceback
import requests
from requests.exceptions import RequestException
import six
from six.moves import configparser
......@@ -56,6 +60,7 @@ class TheiaImport(object):
configp = configparser.SafeConfigParser()
configp.read(cfg)
self.proxy_db = configp.get('slapproxy', 'database_uri')
self.proxy_rest_url = configp.get('slapos', 'master_rest_url') # 200 OK
self.instance_dir = configp.get('slapos', 'instance_root')
mirror_dir = self.mirror_path(self.instance_dir)
partitions = glob.glob(os.path.join(mirror_dir, 'slappart*'))
......@@ -150,7 +155,27 @@ class TheiaImport(object):
print(msg)
raise Exception(msg)
def wait_for_proxy(self):
timeout = 10
sleep = 20
url = self.proxy_rest_url
for i in range(10):
try:
self.log(
'- GET proxy %s with timeout %d (attempt %d)' % (url, timeout, i))
response = requests.get(url, verify=True, timeout=timeout)
except RequestException:
if i == 9:
raise
else:
self.log('- Sleep %ds before retrying' % sleep)
time.sleep(sleep)
else:
response.raise_for_status()
break
def log(self, msg):
msg = time.strftime("%Y-%B-%d %H:%M:%S - ") + msg
print(msg)
self.logs.append(msg)
......@@ -214,6 +239,9 @@ class TheiaImport(object):
self.log('Start slapproxy again')
self.supervisorctl('start', 'slapos-proxy')
self.log('Wait until slapproxy is available')
self.wait_for_proxy()
self.log('Reformat partitions')
self.slapos('node', 'format', '--now')
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment