Skip to content

slapos
slapos
Commit 30dd0182 authored by Xavier Thompson's avatar Xavier Thompson
Browse files
Options

software/theia: Refactor tests

parent 890a40db
Hide whitespace changes
Inline Side-by-side
Showing with 94 additions and 97 deletions
software/theia/test/project_tests.py
View file @ 30dd0182
......@@ -62,10 +62,9 @@ class ERP5Mixin(object):
_test_software_url = erp5_software_release_url
_connexion_parameters_regex = re.compile(r"{.*}", re.DOTALL)
def _getERP5ConnexionParameters(self, software_type='export'):
slapos = self._getSlapos(software_type)
out = subprocess.check_output(
(slapos, 'request', 'test_instance', self._test_software_url),
def _getERP5ConnexionParameters(self, instance_type='export'):
out = self.captureSlapos(
'request', 'test_instance', self._test_software_url,
stderr=subprocess.STDOUT,
)
print(out)
......@@ -110,10 +109,10 @@ class ERP5Mixin(object):
raise Exception("Found several partitions for ERP5 %s" % servicename)
return found.pop()
def _getERP5PartitionPath(self, software_type, servicename, *paths):
def _getERP5PartitionPath(self, instance_type, servicename, *paths):
partition = self._getERP5Partition(servicename)
return self._getPartitionPath(
software_type, 'srv', 'runner', 'instance', partition, *paths)
return self.getPartitionPath(
instance_type, 'srv', 'runner', 'instance', partition, *paths)
class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
......@@ -161,16 +160,15 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
# Update ERP5 parameters
print('Requesting ERP5 with parameters %s' % params)
slapos = self._getSlapos()
subprocess.check_call((slapos, 'request', 'test_instance', self._test_software_url, '--parameters', params))
self.checkSlapos('request', 'test_instance', self._test_software_url, '--parameters', params)
# Process twice to propagate parameter changes
for _ in range(2):
subprocess.check_call((slapos, 'node', 'instance'))
self.checkSlapos('node', 'instance')
# Restart cron (actually all) services to let them take the new date into account
# XXX this should not be required, updating ERP5 parameters should be enough
subprocess.call((slapos, 'node', 'restart', 'all'))
self.callSlapos('node', 'restart', 'all')
# Wait until after the programmed backup date, and a bit more
t = (soon - datetime.now()).total_seconds()
......@@ -213,9 +211,8 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
self.assertNotIn(self._erp5_new_title, out)
# Stop all services
slapos = self._getSlapos()
print("Stop all services")
subprocess.call((slapos, 'node', 'stop', 'all'))
self.callSlapos('node', 'stop', 'all')
# Manually restore mariadb from backup
mariadb_restore_script = os.path.join(mariadb_partition, 'bin', 'restore-from-backup')
......
software/theia/test/test.py
View file @ 30dd0182
......@@ -53,7 +53,7 @@ class TheiaTestCase(SlapOSInstanceTestCase):
__partition_reference__ = 'T' # for supervisord sockets in included slapos
@classmethod
def _getPath(cls, *components):
def getPath(cls, *components):
return os.path.join(cls.computer_partition_root_path, *components)
@classmethod
......@@ -61,7 +61,7 @@ class TheiaTestCase(SlapOSInstanceTestCase):
try:
return cls._theia_slapos
except AttributeError:
cls._theia_slapos = slapos = cls._getPath('srv', 'runner', 'bin', 'slapos')
cls._theia_slapos = slapos = cls.getPath('srv', 'runner', 'bin', 'slapos')
return slapos
@classmethod
......@@ -70,12 +70,11 @@ class TheiaTestCase(SlapOSInstanceTestCase):
@classmethod
def checkSlapos(cls, *command, **kwargs):
kwargs['universal_newlines'] = True
return subprocess.check_call((cls._getSlapos(),) + command, **kwargs)
@classmethod
def captureSlapos(cls, *command, **kwargs):
kwargs['universal_newlines'] = True
kwargs.setdefault('universal_newlines', kwargs.pop('text', None))
return subprocess.check_output((cls._getSlapos(),) + command, **kwargs)
@classmethod
......@@ -147,7 +146,7 @@ class TestTheia(TheiaTestCase):
# there's a public folder to serve file
with open('{}/srv/frontend-static/public/test_file'.format(
self.computer_partition_root_path), 'w') as f:
self.getPath()), 'w') as f:
f.write("hello")
resp = self.get(urljoin(authenticated_url, '/public/'))
self.assertIn('test_file', resp.text)
......@@ -171,10 +170,9 @@ class TestTheia(TheiaTestCase):
self.assertIn('ipv6', self.connection_parameters)
def test_theia_slapos(self):
home = self.getPath()
# Make sure we can use the shell and the integrated slapos command
process = pexpect.spawnu(
'{}/bin/theia-shell'.format(self.computer_partition_root_path),
env={'HOME': self.computer_partition_root_path})
process = pexpect.spawnu(home + '/bin/theia-shell', env={'HOME': home})
# use a large enough terminal so that slapos proxy show table fit in the screen
process.setwinsize(5000, 5000)
......@@ -216,10 +214,11 @@ class TestTheia(TheiaTestCase):
process.wait()
def test_theia_shell_execute_tasks(self):
home = self.getPath()
# shell needs to understand -c "command" arguments for theia tasks feature
test_file = '{}/test file'.format(self.computer_partition_root_path)
test_file = home + '/test file'
subprocess.check_call([
'{}/bin/theia-shell'.format(self.computer_partition_root_path),
home + '/bin/theia-shell',
'-c',
'touch "{}"'.format(test_file)
])
......@@ -227,7 +226,7 @@ class TestTheia(TheiaTestCase):
def test_theia_request_script(self):
script_path = os.path.join(
self.computer_partition_root_path,
self.getPath(),
'srv',
'project',
'request-script-example.sh',
......@@ -235,18 +234,15 @@ class TestTheia(TheiaTestCase):
self.assertTrue(os.path.exists(script_path))
def test_slapos_cli(self):
slapos = self._getSlapos()
proxy_show_output = subprocess.check_output((slapos, 'proxy', 'show'))
self.assertIn(b'slaprunner', proxy_show_output)
computer_list_output = subprocess.check_output((slapos, 'computer', 'list'))
self.assertIn(b'slaprunner', computer_list_output)
self.assertIn(b'slaprunner', self.captureSlapos('proxy', 'show'))
self.assertIn(b'slaprunner', self.captureSlapos('computer', 'list'))
class TestTheiaEmbeddedSlapOSShutdown(TheiaTestCase):
def test_stopping_instance_stops_embedded_slapos(self):
embedded_slapos_supervisord_socket = _getSupervisordSocketPath(
os.path.join(
self.computer_partition_root_path,
self.getPath(),
'srv',
'runner',
'instance',
......@@ -372,7 +368,7 @@ class TestTheiaEnv(TheiaTestCase):
"""Make sure environment variables are the same whether we use shell or supervisor services.
"""
# The path of the env.json file expected to be generated by building the dummy software release
env_json_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'software', 'env.json')
env_json_path = self.getPath('srv', 'runner', 'software', 'env.json')
# Get the pid of the theia process from the test node's instance-supervisord
with self.slap.instance_supervisor_rpc as supervisor:
......@@ -390,7 +386,7 @@ class TestTheiaEnv(TheiaTestCase):
# Start a theia shell that inherits the environment of the theia process
# This simulates the environment of a shell launched from the browser application
theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.computer_partition_root_path), env=theia_env)
theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.getPath()), env=theia_env)
try:
theia_shell_process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
......@@ -410,7 +406,7 @@ class TestTheiaEnv(TheiaTestCase):
# Note that we have two services, slapos-node-software and slapos-node-software-all
# The later uses --all which is what we want to use here, because the software
# is already installed and we want to install it again, this time from supervisor
embedded_run_path = os.path.join(self.computer_partition_root_path, 'srv', 'runner', 'var', 'run')
embedded_run_path = self.getPath('srv', 'runner', 'var', 'run')
embedded_supervisord_socket_path = _getSupervisordSocketPath(embedded_run_path, self.logger)
with getSupervisorRPC(embedded_supervisord_socket_path) as embedded_supervisor:
previous_stop_time = embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop']
......@@ -443,6 +439,8 @@ class ResilientTheiaMixin(object):
@classmethod
def setUpClass(cls):
super(ResilientTheiaMixin, cls).setUpClass()
# Patch the computer root path to that of the export theia instance
cls.computer_partition_root_path = cls.getPartitionPath('export')
# Add resiliency files to snapshot patterns
cls._save_instance_file_pattern_list += (
'*/srv/export-exitcode-file',
......@@ -452,48 +450,53 @@ class ResilientTheiaMixin(object):
)
@classmethod
def _getPartition(cls, software_type):
def getPartitionId(cls, instance_type):
software_url = cls.getSoftwareURL()
for computer_partition in cls.slap.computer.getComputerPartitionList():
partition_url = computer_partition.getSoftwareRelease()._software_release
partition_type = computer_partition.getType()
if partition_url == software_url and partition_type == software_type:
return computer_partition
raise Exception("Theia %s partition not found" % software_type)
if partition_url == software_url and partition_type == instance_type:
return computer_partition.getId()
raise Exception("Theia %s partition not found" % instance_type)
@classmethod
def getPartitionPath(cls, instance_type='export', *paths):
return os.path.join(cls.slap._instance_root, cls.getPartitionId(instance_type), *paths)
@classmethod
def _getSlapos(cls, instance_type='export'):
return cls.getPartitionPath(instance_type, 'srv', 'runner', 'bin', 'slapos')
@classmethod
def _getPartitionId(cls, software_type):
return cls._getPartition(software_type).getId()
def callSlapos(cls, *command, **kwargs):
instance_type = kwargs.pop('instance_type', 'export')
return subprocess.call((cls._getSlapos(instance_type),) + command, **kwargs)
@classmethod
def _getPartitionPath(cls, software_type, *paths):
return os.path.join(cls.slap._instance_root, cls._getPartitionId(software_type), *paths)
def checkSlapos(cls, *command, **kwargs):
instance_type = kwargs.pop('instance_type', 'export')
return subprocess.check_call((cls._getSlapos(instance_type),) + command, **kwargs)
@classmethod
def _getSlapos(cls, software_type='export'):
return cls._getPartitionPath(software_type, 'srv', 'runner', 'bin', 'slapos')
def captureSlapos(cls, *command, **kwargs):
kwargs.setdefault('universal_newlines', kwargs.pop('text', None))
instance_type = kwargs.pop('instance_type', 'export')
return subprocess.check_output((cls._getSlapos(instance_type),) + command, **kwargs)
@classmethod
def getInstanceSoftwareType(cls):
return 'resilient'
def waitForinstance(self, *args, **kwargs):
@classmethod
def waitForInstance(cls):
# process twice to propagate to all instances
for _ in range(2):
super(ResilientTheiaMixin, self).waitForinstance(*args, **kwargs)
super(ResilientTheiaMixin, cls).waitForInstance()
class TestTheiaResilientInterface(ResilientTheiaMixin, TestTheia):
@classmethod
def setUpClass(cls):
super(TestTheiaResilientInterface, cls).setUpClass()
# Patch the computer root path to that of the export theia instance
cls.computer_partition_root_path = cls._getPartitionPath('export')
pass
class TestTheiaResilientWithEmbeddedInstance(ResilientTheiaMixin, TestTheiaWithEmbeddedInstance):
@classmethod
def setUpClass(cls):
super(TestTheiaResilientWithEmbeddedInstance, cls).setUpClass()
# Patch the computer root path to that of the export theia instance
cls.computer_partition_root_path = cls._getPartitionPath('export')
pass
software/theia/test/test_resiliency.py
View file @ 30dd0182
......@@ -63,11 +63,10 @@ def setUpModule():
class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase):
@classmethod
def _processEmbeddedInstance(cls, retries=0, software_type='export'):
slapos = cls._getSlapos(software_type)
def _processEmbeddedInstance(cls, retries=0, instance_type='export'):
for _ in range(retries):
try:
output = subprocess.check_output((slapos, 'node', 'instance'), stderr=subprocess.STDOUT)
output = cls.captureSlapos('node', 'instance', instance_type=instance_type, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
continue
print(output)
......@@ -77,19 +76,18 @@ class ResilientTheiaTestCase(ResilientTheiaMixin, TheiaTestCase):
# Sleep a bit as an attempt to workaround monitoring boostrap not being ready
print("Wait before running slapos node instance one last time")
time.sleep(120)
subprocess.check_call((slapos, 'node', 'instance'))
cls.callSlapos('node', 'instance', instance_type=instance_type)
@classmethod
def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, software_type='export'):
slapos = cls._getSlapos(software_type)
subprocess.check_call((slapos, 'supply', software_url, 'slaprunner'))
def _deployEmbeddedSoftware(cls, software_url, instance_name, retries=0, instance_type='export'):
cls.callSlapos('supply', software_url, 'slaprunner', instance_type=instance_type)
try:
subprocess.check_output((slapos, 'node', 'software'), stderr=subprocess.STDOUT)
cls.captureSlapos('node', 'software', instance_type=instance_type, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
print(e.output)
raise
subprocess.check_call((slapos, 'request', instance_name, software_url))
cls._processEmbeddedInstance(retries, software_type)
cls.callSlapos('request', instance_name, software_url, instance_type=instance_type)
cls._processEmbeddedInstance(retries, instance_type)
@classmethod
def getInstanceParameterDict(cls):
......@@ -136,16 +134,16 @@ class ResilienceMixin(object):
class ExportAndImportMixin(object):
def getExportExitfile(self):
return self._getPartitionPath('export', 'srv', 'export-exitcode-file')
return self.getPartitionPath('export', 'srv', 'export-exitcode-file')
def getExportErrorfile(self):
return self._getPartitionPath('export', 'srv', 'export-errormessage-file')
return self.getPartitionPath('export', 'srv', 'export-errormessage-file')
def getImportExitfile(self):
return self._getPartitionPath('import', 'srv', 'import-exitcode-file')
return self.getPartitionPath('import', 'srv', 'import-exitcode-file')
def getImportErrorfile(self):
return self._getPartitionPath('import', 'srv', 'import-errormessage-file')
return self.getPartitionPath('import', 'srv', 'import-errormessage-file')
def makedirs(self, path):
try:
......@@ -185,7 +183,7 @@ class ExportAndImportMixin(object):
initial_exitdate = os.path.getmtime(exitfile)
# Call export script manually
theia_export_script = self._getPartitionPath('export', 'bin', 'theia-export-script')
theia_export_script = self.getPartitionPath('export', 'bin', 'theia-export-script')
subprocess.check_call((theia_export_script,), stderr=subprocess.STDOUT)
# Check that the export exitcode file was modified
......@@ -198,8 +196,8 @@ class ExportAndImportMixin(object):
def _doTransfer(self):
# Copy <export>/srv/backup/theia to <import>/srv/backup/theia manually
export_backup_path = self._getPartitionPath('export', 'srv', 'backup', 'theia')
import_backup_path = self._getPartitionPath('import', 'srv', 'backup', 'theia')
export_backup_path = self.getPartitionPath('export', 'srv', 'backup', 'theia')
import_backup_path = self.getPartitionPath('import', 'srv', 'backup', 'theia')
shutil.rmtree(import_backup_path)
shutil.copytree(export_backup_path, import_backup_path)
......@@ -209,7 +207,7 @@ class ExportAndImportMixin(object):
initial_exitdate = os.path.getmtime(exitfile)
# Call the import script manually
theia_import_script = self._getPartitionPath('import', 'bin', 'theia-import-script')
theia_import_script = self.getPartitionPath('import', 'bin', 'theia-import-script')
subprocess.check_call((theia_import_script,), stderr=subprocess.STDOUT)
# Check that the import exitcode file was updated
......@@ -277,11 +275,11 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
os.remove(path)
def customSignatureScript(self, content=None):
custom_script = self._getPartitionPath('export', self.script_relpath)
custom_script = self.getPartitionPath('export', self.script_relpath)
self.customScript(custom_script, content)
def customRestoreScript(self, content=None):
restore_script = self._getPartitionPath('import', 'srv', 'runner-import-restore')
restore_script = self.getPartitionPath('import', 'srv', 'runner-import-restore')
self.customScript(restore_script, content)
return restore_script
......@@ -294,7 +292,7 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
self.customRestoreScript(content=None)
self.cleanupExitfiles()
try:
os.remove(self._getPartitionPath('import', self.signature_relpath))
os.remove(self.getPartitionPath('import', self.signature_relpath))
except OSError:
pass
......@@ -309,13 +307,13 @@ class TestTheiaExportAndImportFailures(ExportAndImportMixin, ResilientTheiaTestC
def test_custom_hash_script(self):
errmsg = 'Bye bye'
self.customSignatureScript(content='>&2 echo "%s"\nexit 1' % errmsg)
custom_script = self._getPartitionPath('export', self.script_relpath)
custom_script = self.getPartitionPath('export', self.script_relpath)
self.assertExportFailure('Compute partitions backup signatures\n ... ERROR !',
'Custom signature script %s failed' % os.path.abspath(custom_script),
'and stderr:\n%s' % errmsg)
def test_signature_mismatch(self):
signature_file = self._getPartitionPath('import', self.signature_relpath)
signature_file = self.getPartitionPath('import', self.signature_relpath)
self.writeFile(signature_file, 'Bogus Hash\n', mode='a')
self.assertImportFailure('ERROR the backup signatures do not match')
......@@ -344,7 +342,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
def _prepareExport(self):
# Copy ./resilience_dummy SR in export theia ~/srv/project/dummy
dummy_target_path = self._getPartitionPath('export', 'srv', 'project', 'dummy')
dummy_target_path = self.getPartitionPath('export', 'srv', 'project', 'dummy')
shutil.copytree(os.path.dirname(dummy_software_url), dummy_target_path)
self._test_software_url = os.path.join(dummy_target_path, 'software.cfg')
......@@ -352,8 +350,8 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
self._deployEmbeddedSoftware(self._test_software_url, 'dummy_instance')
relpath_dummy = os.path.join('srv', 'runner', 'instance', 'slappart0')
self.export_dummy_root = dummy_root = self._getPartitionPath('export', relpath_dummy)
self.import_dummy_root = self._getPartitionPath('import', relpath_dummy)
self.export_dummy_root = dummy_root = self.getPartitionPath('export', relpath_dummy)
self.import_dummy_root = self.getPartitionPath('import', relpath_dummy)
# Check that dummy instance was properly deployed
self.initial_log = self.checkLog(os.path.join(dummy_root, 'log.log'))
......@@ -373,7 +371,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
self.assertTrue(os.path.exists(os.path.join(dummy_root, 'srv', '.backup_identity_script')))
# Remember content of ~/etc in the import theia
self.etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
def _doSync(self):
self._doExport()
......@@ -384,14 +382,13 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
dummy_root = self.import_dummy_root
# Check that the software url is correct
adapted_test_url = self._getPartitionPath('import', 'srv', 'project', 'dummy', 'software.cfg')
proxy_content = subprocess.check_output(
(self._getSlapos('import'), 'proxy', 'show'), universal_newlines=True)
adapted_test_url = self.getPartitionPath('import', 'srv', 'project', 'dummy', 'software.cfg')
proxy_content = self.captureSlapos('proxy', 'show', instance_type='import', text=True)
self.assertIn(adapted_test_url, proxy_content)
self.assertNotIn(self._test_software_url, proxy_content)
# Check that ~/etc still contains everything it did before
etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
self.assertTrue(set(self.etc_listdir).issubset(etc_listdir))
# Check that ~/srv/project was exported
......@@ -401,7 +398,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
self.checkLog(os.path.join(dummy_root, 'log.log'), self.initial_log, newline=None)
# Check that ~/srv/.backup_identity_script was detected and called
signature = self._getPartitionPath(
signature = self.getPartitionPath(
'import', 'srv', 'backup', 'theia', 'slappart0.backup.signature.custom')
self.assertTrue(os.path.exists(signature))
with open(signature) as f:
......@@ -418,7 +415,7 @@ class TestTheiaExportAndImport(ResilienceMixin, ExportAndImportMixin, ResilientT
def _doTakeover(self):
# Start the dummy instance as a sort of fake takeover
subprocess.check_call((self._getSlapos('import'), 'node', 'instance'))
self.callSlapos('node', 'instance', instance_type='import')
def _checkTakeover(self):
# Check that dummy instance was properly re-deployed
......@@ -493,7 +490,7 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
# Run two synchronisations on the same instances
# to make sure everything still works the second time
# Check ~/etc in import theia again
self.etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
self._doSync()
self._checkSync()
......@@ -502,18 +499,18 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
self._deployEmbeddedSoftware(self._test_software_url, 'test_instance', self.test_instance_max_retries)
# Check that there is an export and import instance and get their partition IDs
self.export_id = self._getPartitionId('export')
self.import_id = self._getPartitionId('import')
self.export_id = self.getPartitionId('export')
self.import_id = self.getPartitionId('import')
# Remember content of ~/etc in the import theia
self.etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
self.etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
def _doSync(self):
start = time.time()
# Call exporter script instead of waiting for cron job
# XXX Accelerate cron frequency instead ?
exporter_script = self._getPartitionPath('export', 'bin', 'exporter')
exporter_script = self.getPartitionPath('export', 'bin', 'exporter')
transaction_id = str(int(time.time()))
subprocess.check_call((exporter_script, '--transaction-id', transaction_id))
......@@ -524,7 +521,7 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
def _checkSync(self):
# Check that ~/etc still contains everything it did before
etc_listdir = os.listdir(self._getPartitionPath('import', 'etc'))
etc_listdir = os.listdir(self.getPartitionPath('import', 'etc'))
self.assertTrue(set(self.etc_listdir).issubset(etc_listdir))
def _doTakeover(self):
......@@ -541,9 +538,9 @@ class TestTheiaResilience(ResilienceMixin, TakeoverMixin, ResilientTheiaTestCase
# Check that there is an export, import and frozen instance and get their new partition IDs
import_id = self.import_id
export_id = self.export_id
new_export_id = self._getPartitionId('export')
new_import_id = self._getPartitionId('import')
new_frozen_id = self._getPartitionId('frozen')
new_export_id = self.getPartitionId('export')
new_import_id = self.getPartitionId('import')
new_frozen_id = self.getPartitionId('frozen')
# Check that old export instance is now frozen
self.assertEqual(export_id, new_frozen_id)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7