Commit 8f6917f9 authored by Alain Takoudjou's avatar Alain Takoudjou

allow to format and configure external data storage folder

parent b1b19629
...@@ -218,6 +218,7 @@ class Computer(object): ...@@ -218,6 +218,7 @@ class Computer(object):
"Object representing the computer" "Object representing the computer"
instance_root = None instance_root = None
software_root = None software_root = None
instance_storage_home = None
def __init__(self, reference, interface=None, addr=None, netmask=None, def __init__(self, reference, interface=None, addr=None, netmask=None,
ipv6_interface=None, software_user='slapsoft', ipv6_interface=None, software_user='slapsoft',
...@@ -386,6 +387,7 @@ class Computer(object): ...@@ -386,6 +387,7 @@ class Computer(object):
tap = Tap(partition_dict['reference']) tap = Tap(partition_dict['reference'])
address_list = partition_dict['address_list'] address_list = partition_dict['address_list']
external_storage_list = partition_dict.get('external_storage_list', [])
partition = Partition( partition = Partition(
reference=partition_dict['reference'], reference=partition_dict['reference'],
...@@ -393,6 +395,7 @@ class Computer(object): ...@@ -393,6 +395,7 @@ class Computer(object):
user=user, user=user,
address_list=address_list, address_list=address_list,
tap=tap, tap=tap,
external_storage_list=external_storage_list,
) )
computer.partition_list.append(partition) computer.partition_list.append(partition)
...@@ -464,6 +467,15 @@ class Computer(object): ...@@ -464,6 +467,15 @@ class Computer(object):
os.chown(slapsoft.path, slapsoft_pw.pw_uid, slapsoft_pw.pw_gid) os.chown(slapsoft.path, slapsoft_pw.pw_uid, slapsoft_pw.pw_gid)
os.chmod(self.software_root, 0o755) os.chmod(self.software_root, 0o755)
# get list of instance external storage if exist
instance_external_list = []
if self.instance_storage_home:
# XXX - Hard limit for storage number to 4
for i in range(1, 5):
storage_path = os.path.join(self.instance_storage_home, 'data%s' % i)
if os.path.exists(storage_path):
instance_external_list.append(storage_path)
tap_address_list = [] tap_address_list = []
if alter_network and self.tap_gateway_interface and create_tap: if alter_network and self.tap_gateway_interface and create_tap:
gateway_addr_dict = getIfaceAddressIPv4(self.tap_gateway_interface) gateway_addr_dict = getIfaceAddressIPv4(self.tap_gateway_interface)
...@@ -481,6 +493,8 @@ class Computer(object): ...@@ -481,6 +493,8 @@ class Computer(object):
partition.path = os.path.join(self.instance_root, partition.reference) partition.path = os.path.join(self.instance_root, partition.reference)
partition.user.setPath(partition.path) partition.user.setPath(partition.path)
partition.user.additional_group_list = [slapsoft.name] partition.user.additional_group_list = [slapsoft.name]
partition.external_storage_list = ['%s/%s' % (path, partition.reference)
for path in instance_external_list]
if alter_user: if alter_user:
partition.user.create() partition.user.create()
...@@ -510,6 +524,7 @@ class Computer(object): ...@@ -510,6 +524,7 @@ class Computer(object):
# Reconstructing partition's directory # Reconstructing partition's directory
partition.createPath(alter_user) partition.createPath(alter_user)
partition.createExternalPath(alter_user)
# Reconstructing partition's address # Reconstructing partition's address
# There should be two addresses on each Computer Partition: # There should be two addresses on each Computer Partition:
...@@ -555,7 +570,7 @@ class Computer(object): ...@@ -555,7 +570,7 @@ class Computer(object):
class Partition(object): class Partition(object):
"Represent a computer partition" "Represent a computer partition"
def __init__(self, reference, path, user, address_list, tap): def __init__(self, reference, path, user, address_list, tap, external_storage_list=[]):
""" """
Attributes: Attributes:
reference: String, the name of the partition. reference: String, the name of the partition.
...@@ -563,6 +578,7 @@ class Partition(object): ...@@ -563,6 +578,7 @@ class Partition(object):
user: User, the user linked to this partition. user: User, the user linked to this partition.
address_list: List of associated IP addresses. address_list: List of associated IP addresses.
tap: Tap, the tap interface linked to this partition. tap: Tap, the tap interface linked to this partition.
external_storage_list: Base path list of folder to format for data storage
""" """
self.reference = str(reference) self.reference = str(reference)
...@@ -570,6 +586,7 @@ class Partition(object): ...@@ -570,6 +586,7 @@ class Partition(object):
self.user = user self.user = user
self.address_list = address_list or [] self.address_list = address_list or []
self.tap = tap self.tap = tap
self.external_storage_list = []
def __getinitargs__(self): def __getinitargs__(self):
return (self.reference, self.path, self.user, self.address_list, self.tap) return (self.reference, self.path, self.user, self.address_list, self.tap)
...@@ -589,6 +606,21 @@ class Partition(object): ...@@ -589,6 +606,21 @@ class Partition(object):
os.chown(self.path, owner_pw.pw_uid, owner_pw.pw_gid) os.chown(self.path, owner_pw.pw_uid, owner_pw.pw_gid)
os.chmod(self.path, 0o750) os.chmod(self.path, 0o750)
def createExternalPath(self, alter_user=True):
"""
Create and external directory of the partition, assign to the partition user
and give it the 750 permission. In case if path exists just modifies it.
"""
for path in self.external_storage_list:
storage_path = os.path.abspath(path)
owner = self.user if self.user else User('root')
if not os.path.exists(storage_path):
os.mkdir(storage_path, 0o750)
if alter_user:
owner_pw = pwd.getpwnam(owner.name)
os.chown(storage_path, owner_pw.pw_uid, owner_pw.pw_gid)
os.chmod(storage_path, 0o750)
class User(object): class User(object):
"""User: represent and manipulate a user on the system.""" """User: represent and manipulate a user on the system."""
...@@ -1167,6 +1199,7 @@ def do_format(conf): ...@@ -1167,6 +1199,7 @@ def do_format(conf):
computer.instance_root = conf.instance_root computer.instance_root = conf.instance_root
computer.software_root = conf.software_root computer.software_root = conf.software_root
computer.instance_storage_home = conf.instance_storage_home
conf.logger.info('Updating computer') conf.logger.info('Updating computer')
address = computer.getAddress(conf.create_tap) address = computer.getAddress(conf.create_tap)
computer.address = address['addr'] computer.address = address['addr']
...@@ -1208,6 +1241,7 @@ class FormatConfig(object): ...@@ -1208,6 +1241,7 @@ class FormatConfig(object):
software_user = None software_user = None
tap_gateway_interface = None tap_gateway_interface = None
use_unique_local_address_block = None use_unique_local_address_block = None
instance_storage_home = None
def __init__(self, logger): def __init__(self, logger):
self.logger = logger self.logger = logger
......
...@@ -57,6 +57,8 @@ WATCHDOG_MARK = '-on-watch' ...@@ -57,6 +57,8 @@ WATCHDOG_MARK = '-on-watch'
REQUIRED_COMPUTER_PARTITION_PERMISSION = 0o750 REQUIRED_COMPUTER_PARTITION_PERMISSION = 0o750
CP_STORAGE_FOLDER_NAME = 'DATA'
# XXX not very clean. this is changed when testing # XXX not very clean. this is changed when testing
PROGRAM_PARTITION_TEMPLATE = pkg_resources.resource_stream(__name__, PROGRAM_PARTITION_TEMPLATE = pkg_resources.resource_stream(__name__,
'templates/program_partition_supervisord.conf.in').read() 'templates/program_partition_supervisord.conf.in').read()
...@@ -341,7 +343,8 @@ class Partition(object): ...@@ -341,7 +343,8 @@ class Partition(object):
logger, logger,
certificate_repository_path=None, certificate_repository_path=None,
retention_delay='0', retention_delay='0',
instance_min_free_space=None instance_min_free_space=None,
instance_storage_home=''
): ):
"""Initialisation of class parameters""" """Initialisation of class parameters"""
self.buildout = buildout self.buildout = buildout
...@@ -358,6 +361,7 @@ class Partition(object): ...@@ -358,6 +361,7 @@ class Partition(object):
self.partition_id = partition_id self.partition_id = partition_id
self.server_url = server_url self.server_url = server_url
self.software_release_url = software_release_url self.software_release_url = software_release_url
self.instance_storage_home = instance_storage_home
self.key_file = '' self.key_file = ''
self.cert_file = '' self.cert_file = ''
...@@ -512,6 +516,7 @@ class Partition(object): ...@@ -512,6 +516,7 @@ class Partition(object):
'software_release_url': self.software_release_url, 'software_release_url': self.software_release_url,
'key_file': self.key_file, 'key_file': self.key_file,
'cert_file': self.cert_file, 'cert_file': self.cert_file,
'storage_home': self.instance_storage_home,
} }
open(config_location, 'w').write(buildout_text) open(config_location, 'w').write(buildout_text)
os.chmod(config_location, 0o640) os.chmod(config_location, 0o640)
...@@ -684,12 +689,22 @@ class Partition(object): ...@@ -684,12 +689,22 @@ class Partition(object):
sr_symlink = os.path.join(self.instance_path, 'software_release') sr_symlink = os.path.join(self.instance_path, 'software_release')
if os.path.islink(sr_symlink): if os.path.islink(sr_symlink):
os.unlink(sr_symlink) os.unlink(sr_symlink)
data_base_link = os.path.join(self.instance_path, CP_STORAGE_FOLDER_NAME)
if self.instance_storage_home and os.path.exists(data_base_link) and \
os.path.isdir(data_base_link):
for filename in os.listdir(data_base_link):
data_symlink = os.path.join(data_base_link, filename)
partition_data_path = os.path.join(self.instance_storage_home,
filename, self.partition_id)
if os.path.lexists(data_symlink):
os.unlink(data_symlink)
if os.path.exists(partition_data_path):
self.cleanupFolder(partition_data_path)
self.cleanupFolder(self.instance_path)
# Cleanup all Data storage location of this partition
for root, dirs, file_list in os.walk(self.instance_path):
for directory in dirs:
shutil.rmtree(os.path.join(self.instance_path, directory))
for file in file_list:
os.remove(os.path.join(self.instance_path, file))
if os.path.exists(self.supervisord_partition_configuration_path): if os.path.exists(self.supervisord_partition_configuration_path):
os.remove(self.supervisord_partition_configuration_path) os.remove(self.supervisord_partition_configuration_path)
...@@ -699,6 +714,15 @@ class Partition(object): ...@@ -699,6 +714,15 @@ class Partition(object):
return True return True
def cleanupFolder(self, folder_path):
"""Delete all files and folders in a specified directory
"""
for root, dirs, file_list in os.walk(folder_path):
for directory in dirs:
shutil.rmtree(os.path.join(folder_path, directory))
for file in file_list:
os.remove(os.path.join(folder_path, file))
def fetchInformations(self): def fetchInformations(self):
"""Fetch usage informations with buildout, returns it. """Fetch usage informations with buildout, returns it.
""" """
......
...@@ -232,7 +232,8 @@ def create_slapgrid_object(options, logger): ...@@ -232,7 +232,8 @@ def create_slapgrid_object(options, logger):
# Try to fetch from deprecated argument # Try to fetch from deprecated argument
computer_partition_filter_list=op.get('only-cp', op.get('only_cp')), computer_partition_filter_list=op.get('only-cp', op.get('only_cp')),
software_min_free_space=software_min_free_space, software_min_free_space=software_min_free_space,
instance_min_free_space=instance_min_free_space) instance_min_free_space=instance_min_free_space,
instance_storage_home=op.get('instance_storage_home'))
def check_required_only_partitions(existing, required): def check_required_only_partitions(existing, required):
...@@ -286,6 +287,7 @@ class Slapgrid(object): ...@@ -286,6 +287,7 @@ class Slapgrid(object):
computer_partition_filter_list=None, computer_partition_filter_list=None,
software_min_free_space=None, software_min_free_space=None,
instance_min_free_space=None, instance_min_free_space=None,
instance_storage_home=None,
): ):
"""Makes easy initialisation of class parameters""" """Makes easy initialisation of class parameters"""
# Parses arguments # Parses arguments
...@@ -337,6 +339,10 @@ class Slapgrid(object): ...@@ -337,6 +339,10 @@ class Slapgrid(object):
self.maximum_periodicity = maximum_periodicity self.maximum_periodicity = maximum_periodicity
self.software_min_free_space = software_min_free_space self.software_min_free_space = software_min_free_space
self.instance_min_free_space = instance_min_free_space self.instance_min_free_space = instance_min_free_space
if instance_storage_home:
self.instance_storage_home = os.path.abspath(instance_storage_home)
else:
self.instance_storage_home = ""
def _getWatchdogLine(self): def _getWatchdogLine(self):
invocation_list = [WATCHDOG_PATH] invocation_list = [WATCHDOG_PATH]
...@@ -644,6 +650,7 @@ class Slapgrid(object): ...@@ -644,6 +650,7 @@ class Slapgrid(object):
logger=self.logger, logger=self.logger,
retention_delay=retention_delay, retention_delay=retention_delay,
instance_min_free_space=self.instance_min_free_space, instance_min_free_space=self.instance_min_free_space,
instance_storage_home=self.instance_storage_home,
) )
computer_partition_state = computer_partition.getState() computer_partition_state = computer_partition.getState()
...@@ -1102,6 +1109,7 @@ class Slapgrid(object): ...@@ -1102,6 +1109,7 @@ class Slapgrid(object):
certificate_repository_path=self.certificate_repository_path, certificate_repository_path=self.certificate_repository_path,
buildout=self.buildout, buildout=self.buildout,
logger=self.logger, logger=self.logger,
instance_storage_home=self.instance_storage_home,
) )
local_partition.stop() local_partition.stop()
try: try:
......
...@@ -22,4 +22,7 @@ server_url = %(server_url)s ...@@ -22,4 +22,7 @@ server_url = %(server_url)s
software_release_url = %(software_release_url)s software_release_url = %(software_release_url)s
key_file = %(key_file)s key_file = %(key_file)s
cert_file = %(cert_file)s cert_file = %(cert_file)s
[storage-configuration]
storage-home = %(storage_home)s
# This is end of zc.builodout profile's tail added by slapgrid # This is end of zc.builodout profile's tail added by slapgrid
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment