Merge branch 'tap-nobridge' to master

Conflicts:
	master/bt5/slapos_pdm/SkinTemplateItem/portal_skins/slapos_pdm/SoftwareProduct_getSortedSoftwareReleaseList.xml

master/bt5/slapos_slap_tool/SkinTemplateItem/portal_skins/slapos_slap_tool/Computer_updateFromDict.xml

@@ -65,6 +65,10 @@
     if current_dict in to_add_ip_dict_list:\n
       to_add_ip_dict_list.remove(current_dict)\n
     else:\n
+      # XXX - Only delete if Network interface are supposed to be the same\n
+      if additional_dict.has_key(\'network_interface\'):\n
+        if address.getNetworkInterface(\'\') and additional_dict[\'network_interface\'] != address.getNetworkInterface():\n
+          continue\n
       to_delete_ip_id_list.append(address.getId())\n
 \n
   for address in to_add_ip_dict_list:\n
@@ -114,6 +118,16 @@ for send_partition in computer_dict[\'partition_list\']:\n
     partition.edit(reference=send_partition[\'reference\'])\n
   network_interface = send_partition[\'tap\'][\'name\']\n
   compareAndUpdateAddressList(partition, send_partition[\'address_list\'], {\'network_interface\': network_interface})\n
+  tap_addr_list = []\n
+  additional_dict = {\'network_interface\':  \'route_\' + network_interface}\n
+  if send_partition[\'tap\'].has_key(\'ipv4_addr\') and send_partition[\'tap\'][\'ipv4_addr\']:\n
+    tap_addr_list.append({\n
+            \'addr\': send_partition[\'tap\'][\'ipv4_addr\'],\n
+            \'netmask\': send_partition[\'tap\'][\'ipv4_netmask\']\n
+      })\n
+    additional_dict[\'gateway_ip_address\'] = send_partition[\'tap\'][\'ipv4_gateway\']\n
+    additional_dict[\'network_address\'] = send_partition[\'tap\'][\'ipv4_network\']\n
+  compareAndUpdateAddressList(partition, tap_addr_list, additional_dict)\n
 \n
 # Desactivate all other partitions\n
 for key, value in existing_partition_dict.items():\n

master/bt5/slapos_slap_tool/TestTemplateItem/portal_components/test.erp5.testSlapOSSlapToolComputerUpdateFromDict.py

@@ -131,6 +131,51 @@ class TestSlapOSCoreComputerUpdateFromDict(testSlapOSMixin):
     self.assertEqual(address.getNetworkInterface(), 'bar')
     self.assertEqual(address.getId(), 'default_network_address')
   
+  def test_CreateSinglePartition_TapNetworkInformation(self):
+    partition = self.computer.newContent(
+      reference='foo',
+      portal_type='Computer Partition',
+    )
+    # No address in the empty partition
+    address_list = partition.contentValues(
+        portal_type='Internet Protocol Address')
+    self.assertEqual(len(address_list), 0)
+
+    parameter_dict = {
+      'partition_list': [{
+        'reference': 'foo',
+        'address_list': [{
+          'addr': 'c',
+          'netmask': 'd',
+        }],
+        'tap': {'name': 'bar',
+                'ipv4_addr': 'e',
+                'ipv4_netmask': 'f',
+                'ipv4_network': 'g',
+                'ipv4_gateway': 'h'},
+      }],
+      'address': 'a',
+      'netmask': 'b',
+    }
+    self.computer.Computer_updateFromDict(parameter_dict)
+
+    address_list = partition.contentValues(
+        portal_type='Internet Protocol Address')
+    self.assertEqual(len(address_list), 2)
+    address_list.sort(key=lambda x: {0: 1, 1: 2}[int(x.getId() != 'default_network_address')])
+    address = address_list[0]
+    self.assertEqual(address.getIpAddress(), 'c')
+    self.assertEqual(address.getNetmask(), 'd')
+    self.assertEqual(address.getNetworkInterface(), 'bar')
+    self.assertEqual(address.getId(), 'default_network_address')
+    
+    address1 = address_list[1]
+    self.assertEqual(address1.getIpAddress(), 'e')
+    self.assertEqual(address1.getNetmask(), 'f')
+    self.assertEqual(address1.getNetworkAddress(), 'g')
+    self.assertEqual(address1.getGatewayIpAddress(), 'h')
+    self.assertEqual(address1.getNetworkInterface(), 'route_bar')
+
   def test_CreateMultiplePartitionNetworkInformation(self):
     partition = self.computer.newContent(
       reference='foo',
@@ -173,6 +218,61 @@ class TestSlapOSCoreComputerUpdateFromDict(testSlapOSMixin):
     self.assertEqual(other_address.getNetmask(), 'f')
     self.assertEqual(other_address.getNetworkInterface(), 'bar')
 
+  def test_CreateMultiplePartition_TapNetworkInformation(self):
+    partition = self.computer.newContent(
+      reference='foo',
+      portal_type='Computer Partition',
+    )
+    # No address in the empty partition
+    address_list = partition.contentValues(
+        portal_type='Internet Protocol Address')
+    self.assertEqual(len(address_list), 0)
+
+    parameter_dict = {
+      'partition_list': [{
+        'reference': 'foo',
+        'address_list': [{
+          'addr': 'c',
+          'netmask': 'd',
+          },{
+          'addr': 'e',
+          'netmask': 'f',
+        }],
+        'tap': {'name': 'bar',
+                'ipv4_addr': 'g',
+                'ipv4_netmask': 'h',
+                'ipv4_network': 'i',
+                'ipv4_gateway': 'j'},
+      }],
+      'address': 'a',
+      'netmask': 'b',
+    }
+    self.computer.Computer_updateFromDict(parameter_dict)
+
+    address_list = partition.contentValues(
+        portal_type='Internet Protocol Address')
+    self.assertEqual(len(address_list), 3)
+    default_address = [x for x in address_list \
+        if x.getId() == 'default_network_address'][0]
+    self.assertEqual(default_address.getIpAddress(), 'c')
+    self.assertEqual(default_address.getNetmask(), 'd')
+    self.assertEqual(default_address.getNetworkInterface(), 'bar')
+
+    other_address_list = [x for x in address_list \
+        if x.getId() != 'default_network_address']
+    other_address_list.sort(key=lambda x: {1: 1, 0: 2}[int(x.getNetworkInterface() == 'bar')])
+    other_address = other_address_list[0]
+    self.assertEqual(other_address.getIpAddress(), 'e')
+    self.assertEqual(other_address.getNetmask(), 'f')
+    self.assertEqual(other_address.getNetworkInterface(), 'bar')
+    
+    other_address1 = other_address_list[1]
+    self.assertEqual(other_address1.getIpAddress(), 'g')
+    self.assertEqual(other_address1.getNetmask(), 'h')
+    self.assertEqual(other_address1.getNetworkAddress(), 'i')
+    self.assertEqual(other_address1.getGatewayIpAddress(), 'j')
+    self.assertEqual(other_address1.getNetworkInterface(), 'route_bar')
+
   def test_UpdateSinglePartitionNetworkInformation(self):
     partition = self.computer.newContent(
       reference='foo',
@@ -210,6 +310,55 @@ class TestSlapOSCoreComputerUpdateFromDict(testSlapOSMixin):
     self.assertEqual(address.getNetworkInterface(), 'bar')
     self.assertEqual(address.getId(), 'foo')
 
+  def test_UpdateSinglePartition_TapNetworkInformation(self):
+    partition = self.computer.newContent(
+      reference='foo',
+      portal_type='Computer Partition',
+    )
+    # No address in the empty partition
+    address_list = partition.contentValues(
+        portal_type='Internet Protocol Address')
+    self.assertEqual(len(address_list), 0)
+    address = partition.newContent(
+      id ='foo',
+      portal_type='Internet Protocol Address',
+    )
+
+    parameter_dict = {
+      'partition_list': [{
+        'reference': 'foo',
+        'address_list': [{
+          'addr': 'c',
+          'netmask': 'd',
+        }],
+        'tap': {'name': 'bar',
+                'ipv4_addr': 'e',
+                'ipv4_netmask': 'f',
+                'ipv4_network': 'g',
+                'ipv4_gateway': 'h'},
+      }],
+      'address': 'a',
+      'netmask': 'b',
+    }
+    self.computer.Computer_updateFromDict(parameter_dict)
+
+    address_list = partition.contentValues(
+        portal_type='Internet Protocol Address')
+    self.assertEqual(len(address_list), 2)
+    address_list.sort(key=lambda x: {1: 1, 0: 2}[int(x.getNetworkInterface() == 'bar')])
+    address = address_list[0]
+    self.assertEqual(address.getIpAddress(), 'c')
+    self.assertEqual(address.getNetmask(), 'd')
+    self.assertEqual(address.getNetworkInterface(), 'bar')
+    self.assertEqual(address.getId(), 'foo')
+    
+    other_address = address_list[1]
+    self.assertEqual(other_address.getIpAddress(), 'e')
+    self.assertEqual(other_address.getNetmask(), 'f')
+    self.assertEqual(other_address.getNetworkAddress(), 'g')
+    self.assertEqual(other_address.getGatewayIpAddress(), 'h')
+    self.assertEqual(other_address.getNetworkInterface(), 'route_bar')
+
   def test_UpdateMultiplePartitionNetworkInformation(self):
     partition = self.computer.newContent(
       reference='foo',
@@ -258,6 +407,143 @@ class TestSlapOSCoreComputerUpdateFromDict(testSlapOSMixin):
     self.assertEqual(other_address.getNetmask(), 'f')
     self.assertEqual(other_address.getNetworkInterface(), 'bar')
 
+  def test_UpdateMultiplePartition_TapNetworkInformation(self):
+    partition = self.computer.newContent(
+      reference='foo',
+      portal_type='Computer Partition',
+    )
+    # No address in the empty partition
+    address_list = partition.contentValues(
+        portal_type='Internet Protocol Address')
+    self.assertEqual(len(address_list), 0)
+    other_address = partition.newContent(
+      id ='foo',
+      portal_type='Internet Protocol Address',
+    )
+    other_address2 = partition.newContent(
+      id ='route_foo',
+      portal_type='Internet Protocol Address',
+    )
+    default_address = partition.newContent(
+      id ='default_network_interface',
+      portal_type='Internet Protocol Address',
+    )
+
+    parameter_dict = {
+      'partition_list': [{
+        'reference': 'foo',
+        'address_list': [{
+          'addr': 'c',
+          'netmask': 'd',
+          },{
+          'addr': 'e',
+          'netmask': 'f',
+        }],
+        'tap': {'name': 'bar',
+                'ipv4_addr': 'g',
+                'ipv4_netmask': 'h',
+                'ipv4_network': 'i',
+                'ipv4_gateway': 'j'},
+      }],
+      'address': 'a',
+      'netmask': 'b',
+    }
+    self.computer.Computer_updateFromDict(parameter_dict)
+
+    address_list = partition.contentValues(
+        portal_type='Internet Protocol Address')
+    self.assertEqual(len(address_list), 3)
+
+    # First address should go to the default one
+    self.assertEqual(default_address.getIpAddress(), 'c')
+    self.assertEqual(default_address.getNetmask(), 'd')
+    self.assertEqual(default_address.getNetworkInterface(), 'bar')
+
+    other_address_list = [x for x in address_list \
+        if x.getId() != 'default_network_interface']
+    other_address_list.sort(
+          key=lambda x: {1: 1, 0: 2}[int(x.getNetworkInterface() == 'bar')])
+    other_address = other_address_list[0]
+    self.assertEqual(other_address.getIpAddress(), 'e')
+    self.assertEqual(other_address.getNetmask(), 'f')
+    self.assertEqual(other_address.getNetworkInterface(), 'bar')
+    
+    other_address = other_address_list[1]
+    self.assertEqual(other_address.getIpAddress(), 'g')
+    self.assertEqual(other_address.getNetmask(), 'h')
+    self.assertEqual(other_address.getNetworkAddress(), 'i')
+    self.assertEqual(other_address.getGatewayIpAddress(), 'j')
+    self.assertEqual(other_address.getNetworkInterface(), 'route_bar')
+
+  def test_removePartitionTapNetworkInformation(self):
+    partition = self.computer.newContent(
+      reference='foo',
+      portal_type='Computer Partition',
+    )
+    # No address in the empty partition
+    address_list = partition.contentValues(
+        portal_type='Internet Protocol Address')
+    self.assertEqual(len(address_list), 0)
+    
+    parameter_dict = {
+      'partition_list': [{
+        'reference': 'foo',
+        'address_list': [{
+          'addr': 'c',
+          'netmask': 'd',
+        }],
+        'tap': {'name': 'bar',
+                'ipv4_addr': 'e',
+                'ipv4_netmask': 'f',
+                'ipv4_network': 'g',
+                'ipv4_gateway': 'h'},
+      }],
+      'address': 'a',
+      'netmask': 'b',
+    }
+    parameter_dict2 = {
+      'partition_list': [{
+        'reference': 'foo',
+        'address_list': [{
+          'addr': 'c',
+          'netmask': 'd',
+        }],
+        'tap': {'name': 'bar'},
+      }],
+      'address': 'a',
+      'netmask': 'b',
+    }
+    
+    self.computer.Computer_updateFromDict(parameter_dict)
+    
+    address_list = partition.contentValues(
+        portal_type='Internet Protocol Address')
+    address_list.sort(
+      key=lambda x: {1: 1, 0: 2}[int(x.getId() == 'default_network_address')])
+    self.assertEqual(len(address_list), 2)
+    address = address_list[0]
+    self.assertEqual(address.getIpAddress(), 'c')
+    self.assertEqual(address.getNetmask(), 'd')
+    self.assertEqual(address.getNetworkInterface(), 'bar')
+    self.assertEqual(address.getId(), 'default_network_address')
+    
+    other_address = address_list[1]
+    self.assertEqual(other_address.getIpAddress(), 'e')
+    self.assertEqual(other_address.getNetmask(), 'f')
+    self.assertEqual(other_address.getNetworkAddress(), 'g')
+    self.assertEqual(other_address.getGatewayIpAddress(), 'h')
+    self.assertEqual(other_address.getNetworkInterface(), 'route_bar')
+    
+    self.computer.Computer_updateFromDict(parameter_dict2)
+    
+    address_list = partition.contentValues(
+        portal_type='Internet Protocol Address')
+    self.assertEqual(len(address_list), 1)
+    self.assertEqual(address_list[0].getIpAddress(), 'c')
+    self.assertEqual(address_list[0].getNetmask(), 'd')
+    self.assertEqual(address_list[0].getNetworkInterface(), 'bar')
+    self.assertEqual(address_list[0].getId(), 'default_network_address')
+
   def test_RemoveSinglePartitionNetworkInformation(self):
     partition = self.computer.newContent(
       reference='foo',

master/bt5/slapos_slap_tool/TestTemplateItem/portal_components/test.erp5.testSlapOSSlapToolComputerUpdateFromDict.xml

@@ -6,10 +6,22 @@
     </pickle>
     <pickle>
       <dictionary>
+        <item>
+            <key> <string>_recorded_property_dict</string> </key>
+            <value>
+              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+            </value>
+        </item>
         <item>
             <key> <string>default_reference</string> </key>
             <value> <string>testSlapOSSlapToolComputerUpdateFromDict</string> </value>
         </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
         <item>
             <key> <string>id</string> </key>
             <value> <string>test.erp5.testSlapOSSlapToolComputerUpdateFromDict</string> </value>
@@ -34,10 +46,11 @@
             <key> <string>text_content_warning_message</string> </key>
             <value>
               <tuple>
-                <string>W:274,  4: Unused variable \'default_address\' (unused-variable)</string>
-                <string>W:270,  4: Unused variable \'other_address\' (unused-variable)</string>
-                <string>W:315,  4: Unused variable \'default_address\' (unused-variable)</string>
-                <string>W:311,  4: Unused variable \'other_address\' (unused-variable)</string>
+                <string>W:423,  4: Unused variable \'other_address2\' (unused-variable)</string>
+                <string>W:560,  4: Unused variable \'default_address\' (unused-variable)</string>
+                <string>W:556,  4: Unused variable \'other_address\' (unused-variable)</string>
+                <string>W:601,  4: Unused variable \'default_address\' (unused-variable)</string>
+                <string>W:597,  4: Unused variable \'other_address\' (unused-variable)</string>
               </tuple>
             </value>
         </item>
@@ -48,13 +61,28 @@
         <item>
             <key> <string>workflow_history</string> </key>
             <value>
-              <persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
+              <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
             </value>
         </item>
       </dictionary>
     </pickle>
   </record>
   <record id="2" aka="AAAAAAAAAAI=">
+    <pickle>
+      <global name="PersistentMapping" module="Persistence.mapping"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>data</string> </key>
+            <value>
+              <dictionary/>
+            </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+  <record id="3" aka="AAAAAAAAAAM=">
     <pickle>
       <global name="PersistentMapping" module="Persistence.mapping"/>
     </pickle>
@@ -67,7 +95,7 @@
                 <item>
                     <key> <string>component_validation_workflow</string> </key>
                     <value>
-                      <persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
+                      <persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
                     </value>
                 </item>
               </dictionary>
@@ -76,7 +104,7 @@
       </dictionary>
     </pickle>
   </record>
-  <record id="3" aka="AAAAAAAAAAM=">
+  <record id="4" aka="AAAAAAAAAAQ=">
     <pickle>
       <global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
     </pickle>

master/product/Vifib/Tool/SlapTool.py

@@ -1429,9 +1429,17 @@ class SlapTool(BaseTool):
 
     ip_list = []
     for internet_protocol_address in computer_partition.contentValues(portal_type='Internet Protocol Address'):
-      ip_list.append((
+      # XXX - There is new values, and we must keep compatibility
+      address_tuple = (
           internet_protocol_address.getNetworkInterface('').decode("UTF-8"),
-        internet_protocol_address.getIpAddress().decode("UTF-8")))
+          internet_protocol_address.getIpAddress().decode("UTF-8"))
+      if internet_protocol_address.getGatewayIpAddress('') and \
+        internet_protocol_address.getNetmask(''):
+        address_tuple = address_tuple + (
+              internet_protocol_address.getGatewayIpAddress().decode("UTF-8"),
+              internet_protocol_address.getNetmask().decode("UTF-8"),
+              internet_protocol_address.getNetworkAddress('').decode("UTF-8"))
+      ip_list.append(address_tuple)
 
     slave_instance_list = []
     if (software_instance.getPortalType() == "Software Instance"):

slapos/format.py

@@ -152,6 +152,36 @@ def netmaskToPrefixIPv6(netmask):
   return netaddr.strategy.ipv6.netmask_to_prefix[
           netaddr.strategy.ipv6.str_to_int(netmask)]
 
+def getIfaceAddressIPv4(iface):
+  """return dict containing ipv4 address netmask, network and broadcast address 
+  of interface"""
+  if not iface in netifaces.interfaces():
+    raise ValueError('Could not find interface called %s to use as gateway ' \
+                      'for tap network' % iface)
+  try:
+    addresses_list = netifaces.ifaddresses(iface)[socket.AF_INET]
+    if len (addresses_list) > 0:
+      
+      addresses = addresses_list[0].copy()
+      addresses['network'] = str(netaddr.IPNetwork('%s/%s' % (addresses['addr'],
+                                          addresses['netmask'])).cidr.network)
+      return addresses
+    else:
+      return {}
+  except KeyError:
+    raise KeyError('Could not find IPv4 adress on interface %s.' % iface)
+
+def getIPv4SubnetAddressRange(ip_address, mask, size):
+  """Check if a given ipaddress can be used to create 'size' 
+  host ip address, then return list of ip address in the subnet"""
+  ip = netaddr.IPNetwork('%s/%s' % (ip_address, mask))
+  # Delete network and default ip_address from the list
+  ip_list = [x for x in sorted(list(ip))
+              if str(x) != ip_address and x.value != ip.cidr.network.value]
+  if len(ip_list) < size:
+    raise ValueError('Could not create %s tap interfaces from address %s.' % (
+              size, ip_address))
+  return ip_list
 
 def _getDict(obj):
   """
@@ -188,9 +218,11 @@ class Computer(object):
   "Object representing the computer"
   instance_root = None
   software_root = None
+  instance_storage_home = None
 
   def __init__(self, reference, interface=None, addr=None, netmask=None,
-               ipv6_interface=None, software_user='slapsoft'):
+               ipv6_interface=None, software_user='slapsoft',
+               tap_gateway_interface=None):
     """
     Attributes:
       reference: String, the reference of the computer.
@@ -203,6 +235,7 @@ class Computer(object):
     self.netmask = netmask
     self.ipv6_interface = ipv6_interface
     self.software_user = software_user
+    self.tap_gateway_interface = tap_gateway_interface
 
   def __getinitargs__(self):
     return (self.reference, self.interface)
@@ -317,7 +350,7 @@ class Computer(object):
       archive.writestr(saved_filename, xml_content, zipfile.ZIP_DEFLATED)
 
   @classmethod
-  def load(cls, path_to_xml, reference, ipv6_interface):
+  def load(cls, path_to_xml, reference, ipv6_interface, tap_gateway_interface):
     """
     Create a computer object from a valid xml file.
 
@@ -338,6 +371,7 @@ class Computer(object):
         netmask=dumped_dict['netmask'],
         ipv6_interface=ipv6_interface,
         software_user=dumped_dict.get('software_user', 'slapsoft'),
+        tap_gateway_interface=tap_gateway_interface,
     )
 
     for partition_dict in dumped_dict['partition_list']:
@@ -349,10 +383,15 @@ class Computer(object):
 
       if partition_dict['tap']:
         tap = Tap(partition_dict['tap']['name'])
+        tap.ipv4_addr = partition_dict['tap'].get('ipv4_addr', '')
+        tap.ipv4_netmask = partition_dict['tap'].get('ipv4_netmask', '')
+        tap.ipv4_gateway = partition_dict['tap'].get('ipv4_gateway', '')
+        tap.ipv4_network = partition_dict['tap'].get('ipv4_network', '')
       else:
         tap = Tap(partition_dict['reference'])
 
       address_list = partition_dict['address_list']
+      external_storage_list = partition_dict.get('external_storage_list', [])
 
       partition = Partition(
           reference=partition_dict['reference'],
@@ -360,6 +399,7 @@ class Computer(object):
           user=user,
           address_list=address_list,
           tap=tap,
+          external_storage_list=external_storage_list,
       )
 
       computer.partition_list.append(partition)
@@ -431,6 +471,23 @@ class Computer(object):
       os.chown(slapsoft.path, slapsoft_pw.pw_uid, slapsoft_pw.pw_gid)
     os.chmod(self.software_root, 0o755)
 
+    # get list of instance external storage if exist
+    instance_external_list = []
+    if self.instance_storage_home:
+      # XXX - Hard limit for storage number to 4
+      for i in range(1, 5):
+        storage_path = os.path.join(self.instance_storage_home, 'data%s' % i)
+        if os.path.exists(storage_path):
+          instance_external_list.append(storage_path)
+
+    tap_address_list = []
+    if alter_network and self.tap_gateway_interface and create_tap:
+      gateway_addr_dict = getIfaceAddressIPv4(self.tap_gateway_interface)
+      tap_address_list = getIPv4SubnetAddressRange(gateway_addr_dict['addr'],
+                              gateway_addr_dict['netmask'],
+                              len(self.partition_list))
+      assert(len(self.partition_list) <= len(tap_address_list))
+
     if alter_network:
       self._speedHackAddAllOldIpsToInterface()
 
@@ -440,6 +497,8 @@ class Computer(object):
         partition.path = os.path.join(self.instance_root, partition.reference)
         partition.user.setPath(partition.path)
         partition.user.additional_group_list = [slapsoft.name]
+        partition.external_storage_list = ['%s/%s' % (path, partition.reference)
+                                            for path in instance_external_list]
         if alter_user:
           partition.user.create()
 
@@ -456,11 +515,23 @@ class Computer(object):
             partition.tap.createWithOwner(owner, attach_to_tap=True)
           else:
             partition.tap.createWithOwner(owner)
-
+          # If tap_gateway_interface is specified, we don't add tap to bridge
+          # but we create route for this tap
+          if not self.tap_gateway_interface:
             self.interface.addTap(partition.tap)
+          else:
+            next_ipv4_addr = '%s' % tap_address_list.pop(0)
+            if not partition.tap.ipv4_addr:
+              # define new ipv4 address for this tap
+              partition.tap.ipv4_addr = next_ipv4_addr
+              partition.tap.ipv4_netmask = gateway_addr_dict['netmask']
+              partition.tap.ipv4_gateway = gateway_addr_dict['addr']
+              partition.tap.ipv4_network = gateway_addr_dict['network']
+            partition.tap.createRoutes()
 
         # Reconstructing partition's directory
         partition.createPath(alter_user)
+        partition.createExternalPath(alter_user)
 
         # Reconstructing partition's address
         # There should be two addresses on each Computer Partition:
@@ -506,7 +577,7 @@ class Computer(object):
 class Partition(object):
   "Represent a computer partition"
 
-  def __init__(self, reference, path, user, address_list, tap):
+  def __init__(self, reference, path, user, address_list, tap, external_storage_list=[]):
     """
     Attributes:
       reference: String, the name of the partition.
@@ -514,6 +585,7 @@ class Partition(object):
       user: User, the user linked to this partition.
       address_list: List of associated IP addresses.
       tap: Tap, the tap interface linked to this partition.
+      external_storage_list: Base path list of folder to format for data storage
     """
 
     self.reference = str(reference)
@@ -521,6 +593,7 @@ class Partition(object):
     self.user = user
     self.address_list = address_list or []
     self.tap = tap
+    self.external_storage_list = []
 
   def __getinitargs__(self):
     return (self.reference, self.path, self.user, self.address_list, self.tap)
@@ -540,6 +613,21 @@ class Partition(object):
       os.chown(self.path, owner_pw.pw_uid, owner_pw.pw_gid)
     os.chmod(self.path, 0o750)
 
+  def createExternalPath(self, alter_user=True):
+    """
+    Create and external directory of the partition, assign to the partition user
+    and give it the 750 permission. In case if path exists just modifies it.
+    """
+
+    for path in self.external_storage_list:
+      storage_path = os.path.abspath(path)
+      owner = self.user if self.user else User('root')
+      if not os.path.exists(storage_path):
+        os.mkdir(storage_path, 0o750)
+      if alter_user:
+        owner_pw = pwd.getpwnam(owner.name)
+        os.chown(storage_path, owner_pw.pw_uid, owner_pw.pw_gid)
+      os.chmod(storage_path, 0o750)
 
 class User(object):
   """User: represent and manipulate a user on the system."""
@@ -617,9 +705,16 @@ class Tap(object):
     """
     Attributes:
         tap_name: String, the name of the tap interface.
+        ipv4_address: String, local ipv4 to route to this tap
+        ipv4_network: String, netmask to use when configure route for this tap
+        gateway_ipv4: String, ipv4 of gateway to be used to reach local network
     """
 
     self.name = str(tap_name)
+    self.ipv4_addr = ""
+    self.ipv4_netmask = ""
+    self.ipv4_gateway = ""
+    self.ipv4_network = ""
 
   def __getinitargs__(self):
     return (self.name,)
@@ -698,6 +793,19 @@ class Tap(object):
     if attach_to_tap:
       threading.Thread(target=self.attach).start()
 
+  def createRoutes(self):
+    """
+    Configure ipv4 route to reach this interface from local network
+    """
+    if self.ipv4_addr:
+      # Check if this route exits
+      code, result = callAndRead(['ip', 'route', 'show', self.ipv4_addr])
+      if code == 0 and self.ipv4_addr in result and self.name in result:
+        return
+      callAndRead(['route', 'add', '-host', self.ipv4_addr, 'dev', self.name])
+    else:
+      raise ValueError("%s should not be empty. No ipv4 address assigned to %s" %
+                         (self.ipv4_addr, self.name))
 
 class Interface(object):
   """Represent a network interface on the system"""
@@ -980,6 +1088,7 @@ def parse_computer_definition(conf, definition_path):
       netmask=netmask,
       ipv6_interface=conf.ipv6_interface,
       software_user=computer_definition.get('computer', 'software_user'),
+      tap_gateway_interface=conf.tap_gateway_interface,
   )
   partition_list = []
   for partition_number in range(int(conf.partition_amount)):
@@ -1011,7 +1120,8 @@ def parse_computer_xml(conf, xml_path):
     conf.logger.debug('Loading previous computer data from %r' % xml_path)
     computer = Computer.load(xml_path,
                              reference=conf.computer_id,
-                             ipv6_interface=conf.ipv6_interface)
+                             ipv6_interface=conf.ipv6_interface,
+                             tap_gateway_interface=conf.tap_gateway_interface)
     # Connect to the interface defined by the configuration
     computer.interface = interface
   else:
@@ -1024,6 +1134,7 @@ def parse_computer_xml(conf, xml_path):
       netmask=None,
       ipv6_interface=conf.ipv6_interface,
       software_user=conf.software_user,
+      tap_gateway_interface=conf.tap_gateway_interface,
     )
 
   partition_amount = int(conf.partition_amount)
@@ -1095,6 +1206,7 @@ def do_format(conf):
 
   computer.instance_root = conf.instance_root
   computer.software_root = conf.software_root
+  computer.instance_storage_home = conf.instance_storage_home
   conf.logger.info('Updating computer')
   address = computer.getAddress(conf.create_tap)
   computer.address = address['addr']
@@ -1134,7 +1246,9 @@ class FormatConfig(object):
   output_definition_file = None
   dry_run = None
   software_user = None
+  tap_gateway_interface = None
   use_unique_local_address_block = None
+  instance_storage_home = None
 
   def __init__(self, logger):
     self.logger = logger
@@ -1202,6 +1316,8 @@ class FormatConfig(object):
       self.software_user = 'slapsoft'
     if self.create_tap is None:
       self.create_tap = True
+    if self.tap_gateway_interface is None:
+      self.tap_gateway_interface = ''
     if self.use_unique_local_address_block is None:
       self.use_unique_local_address_block = False
 
@@ -1225,6 +1341,8 @@ class FormatConfig(object):
         self.checkRequiredBinary(['groupadd', 'useradd', 'usermod'])
       if self.create_tap:
         self.checkRequiredBinary([['tunctl', '-d']])
+        if self.tap_gateway_interface:
+          self.checkRequiredBinary(['route'])
       if self.alter_network:
         self.checkRequiredBinary(['ip'])
 
@@ -1292,3 +1410,4 @@ def tracing_monkeypatch(conf):
     conf.logger.debug(' '.join(argument_list))
     return dry_callAndRead(argument_list, raise_on_error)
   callAndRead = logging_callAndRead
+

slapos/grid/SlapObject.py

@@ -57,6 +57,8 @@ WATCHDOG_MARK = '-on-watch'
 
 REQUIRED_COMPUTER_PARTITION_PERMISSION = 0o750
 
+CP_STORAGE_FOLDER_NAME = 'DATA'
+
 # XXX not very clean. this is changed when testing
 PROGRAM_PARTITION_TEMPLATE = pkg_resources.resource_stream(__name__,
             'templates/program_partition_supervisord.conf.in').read()
@@ -341,7 +343,8 @@ class Partition(object):
                logger,
                certificate_repository_path=None,
                retention_delay='0',
-               instance_min_free_space=None
+               instance_min_free_space=None,
+               instance_storage_home=''
                ):
     """Initialisation of class parameters"""
     self.buildout = buildout
@@ -358,6 +361,7 @@ class Partition(object):
     self.partition_id = partition_id
     self.server_url = server_url
     self.software_release_url = software_release_url
+    self.instance_storage_home = instance_storage_home
 
     self.key_file = ''
     self.cert_file = ''
@@ -512,6 +516,7 @@ class Partition(object):
             'software_release_url': self.software_release_url,
             'key_file': self.key_file,
             'cert_file': self.cert_file,
+            'storage_home': self.instance_storage_home,
         }
     open(config_location, 'w').write(buildout_text)
     os.chmod(config_location, 0o640)
@@ -684,12 +689,22 @@ class Partition(object):
       sr_symlink = os.path.join(self.instance_path, 'software_release')
       if os.path.islink(sr_symlink):
         os.unlink(sr_symlink)
+      data_base_link = os.path.join(self.instance_path, CP_STORAGE_FOLDER_NAME)
+      if self.instance_storage_home and os.path.exists(data_base_link) and \
+                                os.path.isdir(data_base_link):
+        for filename in os.listdir(data_base_link):
+          data_symlink = os.path.join(data_base_link, filename)
+          partition_data_path = os.path.join(self.instance_storage_home,
+                                                    filename, self.partition_id)
+          if os.path.lexists(data_symlink):
+            os.unlink(data_symlink)
+          if os.path.exists(partition_data_path):
+            self.cleanupFolder(partition_data_path)
+
+      self.cleanupFolder(self.instance_path)
+      
+      # Cleanup all Data storage location of this partition
       
-      for root, dirs, file_list in os.walk(self.instance_path):
-        for directory in dirs:
-          shutil.rmtree(os.path.join(self.instance_path, directory))
-        for file in file_list:
-          os.remove(os.path.join(self.instance_path, file))
 
       if os.path.exists(self.supervisord_partition_configuration_path):
         os.remove(self.supervisord_partition_configuration_path)
@@ -699,6 +714,15 @@ class Partition(object):
 
     return True
 
+  def cleanupFolder(self, folder_path):
+    """Delete all files and folders in a specified directory
+    """
+    for root, dirs, file_list in os.walk(folder_path):
+      for directory in dirs:
+        shutil.rmtree(os.path.join(folder_path, directory))
+      for file in file_list:
+        os.remove(os.path.join(folder_path, file))
+
   def fetchInformations(self):
     """Fetch usage informations with buildout, returns it.
     """

slapos/grid/slapgrid.py

@@ -232,7 +232,8 @@ def create_slapgrid_object(options, logger):
                   # Try to fetch from deprecated argument
                   computer_partition_filter_list=op.get('only-cp', op.get('only_cp')),
                   software_min_free_space=software_min_free_space,
-                  instance_min_free_space=instance_min_free_space)
+                  instance_min_free_space=instance_min_free_space,
+                  instance_storage_home=op.get('instance_storage_home'))
 
 
 def check_required_only_partitions(existing, required):
@@ -286,6 +287,7 @@ class Slapgrid(object):
                computer_partition_filter_list=None,
                software_min_free_space=None,
                instance_min_free_space=None,
+               instance_storage_home=None,
                ):
     """Makes easy initialisation of class parameters"""
     # Parses arguments
@@ -337,6 +339,10 @@ class Slapgrid(object):
     self.maximum_periodicity = maximum_periodicity
     self.software_min_free_space = software_min_free_space
     self.instance_min_free_space = instance_min_free_space
+    if instance_storage_home:
+      self.instance_storage_home = os.path.abspath(instance_storage_home)
+    else:
+      self.instance_storage_home = ""
 
   def _getWatchdogLine(self):
     invocation_list = [WATCHDOG_PATH]
@@ -644,6 +650,7 @@ class Slapgrid(object):
         logger=self.logger,
         retention_delay=retention_delay,
         instance_min_free_space=self.instance_min_free_space,
+        instance_storage_home=self.instance_storage_home,
       )
       computer_partition_state = computer_partition.getState()
 
@@ -1102,6 +1109,7 @@ class Slapgrid(object):
             certificate_repository_path=self.certificate_repository_path,
             buildout=self.buildout,
             logger=self.logger,
+            instance_storage_home=self.instance_storage_home,
           )
           local_partition.stop()
           try:

slapos/grid/templates/buildout-tail.cfg.in

@@ -22,4 +22,7 @@ server_url = %(server_url)s
 software_release_url = %(software_release_url)s
 key_file = %(key_file)s
 cert_file = %(cert_file)s
+
+[storage-configuration]
+storage-home = %(storage_home)s
 # This is end of zc.builodout profile's tail added by slapgrid

slapos/tests/slapformat.py

@@ -94,6 +94,10 @@ class FakeCallAndRead:
     elif argument_list[:3] == ['ip', 'addr', 'list'] or \
          argument_list[:4] == ['ip', '-6', 'addr', 'list']:
       retval = 0, str(INTERFACE_DICT)
+    elif argument_list[:3] == ['ip', 'route', 'show']:
+      retval = 0, 'OK'
+    elif argument_list[:3] == ['route', 'add', '-host']:
+      retval = 0, 'OK'
     self.external_command_list.append(' '.join(argument_list))
     return retval
 
@@ -441,6 +445,56 @@ class TestComputer(SlapformatMixin):
     ],
       self.fakeCallAndRead.external_command_list)
 
+  def test_construct_prepared_tap_no_alter_user(self):
+    computer = slapos.format.Computer('computer',
+      interface=slapos.format.Interface(logger=self.test_result,
+                                        name='iface',
+                                        ipv4_local_network='127.0.0.1/16'),
+                                        tap_gateway_interface='eth1')
+    computer.instance_root = '/instance_root'
+    computer.software_root = '/software_root'
+    partition = slapos.format.Partition('partition', '/part_path',
+      slapos.format.User('testuser'), [], None)
+    global USER_LIST
+    USER_LIST = ['testuser']
+    partition.tap = slapos.format.Tap('tap')
+    computer.partition_list = [partition]
+    global INTERFACE_DICT
+    INTERFACE_DICT['iface'] = {
+      socket.AF_INET: [{'addr': '192.168.242.77', 'broadcast': '127.0.0.1',
+        'netmask': '255.255.255.0'}],
+      socket.AF_INET6: [{'addr': '2a01:e35:2e27::e59c', 'netmask': 'ffff:ffff:ffff:ffff::'}]
+    }
+    INTERFACE_DICT['eth1'] = {
+      socket.AF_INET: [{'addr': '10.8.0.1', 'broadcast': '10.8.0.254',
+        'netmask': '255.255.255.0'}]
+    }
+
+    computer.construct(alter_user=False)
+    self.assertEqual([
+      "makedirs('/instance_root', 493)",
+      "makedirs('/software_root', 493)",
+      "chmod('/software_root', 493)",
+      "mkdir('/instance_root/partition', 488)",
+      "chmod('/instance_root/partition', 488)"
+    ],
+      self.test_result.bucket)
+    self.assertEqual([
+        'ip addr list iface',
+        'tunctl -t tap -u testuser',
+        'ip link set tap up',
+        'ip route show 10.8.0.2',
+        'route add -host 10.8.0.2 dev tap',
+        'ip addr add ip/255.255.255.255 dev iface',
+        'ip addr add ip/ffff:ffff:ffff:ffff:: dev iface',
+        'ip -6 addr list iface'
+      ],
+      self.fakeCallAndRead.external_command_list)
+    self.assertEqual(partition.tap.ipv4_addr, '10.8.0.2')
+    self.assertEqual(partition.tap.ipv4_netmask, '255.255.255.0')
+    self.assertEqual(partition.tap.ipv4_gateway, '10.8.0.1')
+    self.assertEqual(partition.tap.ipv4_network, '10.8.0.0')
+
   @unittest.skip("Not implemented")
   def test_construct_prepared_no_alter_network(self):
     computer = slapos.format.Computer('computer',