share rmtree from cli/prune with slapgrid and use it to drop partition

See merge request !227

slapos/cli/prune.py

@@ -34,6 +34,7 @@ import six.moves.configparser as configparser
 
 from slapos.cli.config import ConfigCommand
 from slapos.grid.slapgrid import merged_options
+from slapos.util import rmtree
 
 
 class PruneCommand(ConfigCommand):
@@ -229,53 +230,3 @@ def getUsageSignatureFromSoftwareAndSharedPart(
         with open(shared_signature) as f:
           signatures[shared_signature] = f.read()
   return signatures
-
-
-# XXX copied from https://lab.nexedi.com/nexedi/erp5/blob/31804f683fd36322fb38aeb9654bee70cebe4fdb/erp5/util/testnode/Utils.py
-# TODO: move to shared place or ... isn't there already such an utility function in slapos.core ?
-import shutil
-import errno
-import six
-from six.moves import map
-
-try:
-  PermissionError
-except NameError:  # make pylint happy on python2...
-  PermissionError = Exception
-
-
-def rmtree(path):
-  """Delete a path recursively.
-  Like shutil.rmtree, but supporting the case that some files or folder
-  might have been marked read only.  """
-  def chmod_retry(func, failed_path, exc_info):
-    """Make sure the directories are executable and writable.
-    """
-    # Depending on the Python version, the following items differ.
-    if six.PY3:
-      expected_error_type = PermissionError
-      expected_func = os.lstat
-    else:
-      expected_error_type = OSError
-      expected_func = os.listdir
-    e = exc_info[1]
-    if isinstance(e, expected_error_type):
-      if e.errno == errno.ENOENT:
-        # because we are calling again rmtree on listdir errors, this path might
-        # have been already deleted by the recursive call to rmtree.
-        return
-      if e.errno == errno.EACCES:
-        if func is expected_func:
-          os.chmod(failed_path, 0o700)
-          # corner case to handle errors in listing directories.
-          # https://bugs.python.org/issue8523
-          return shutil.rmtree(failed_path, onerror=chmod_retry)
-        # If parent directory is not writable, we still cannot delete the file.
-        # But make sure not to change the parent of the folder we are deleting.
-        if failed_path != path:
-          os.chmod(os.path.dirname(failed_path), 0o700)
-          return func(failed_path)
-    raise e  # XXX make pylint happy
-
-  shutil.rmtree(path, onerror=chmod_retry)
-# / erp5/util/testnode/Utils.py code

slapos/grid/SlapObject.py

@@ -52,7 +52,7 @@ from slapos.grid.exception import (BuildoutFailedError, WrongPermissionError,
                                    PathDoesNotExistError, DiskSpaceError)
 from slapos.grid.networkcache import download_network_cached, upload_network_cached
 from slapos.human import bytes2human
-from slapos.util import bytes2str
+from slapos.util import bytes2str, rmtree
 
 
 WATCHDOG_MARK = '-on-watch'
@@ -354,17 +354,10 @@ class Software(object):
 
   def destroy(self):
     """Removes software release."""
-    def retry(func, path, exc):
-      # inspired by slapos.buildout hard remover
-      if func == os.path.islink:
-        os.unlink(path)
-      else:
-        os.chmod(path, 0o600)
-        func(path)
     try:
       if os.path.exists(self.software_path):
         self.logger.info('Removing path %r' % self.software_path)
-        shutil.rmtree(self.software_path, onerror=retry)
+        rmtree(self.software_path)
       else:
         self.logger.info('Path %r does not exists, no need to remove.' %
             self.software_path)

slapos/tests/test_slapgrid.py

@@ -397,6 +397,8 @@ class ComputerForTest(object):
         return {'status_code': 200}
       if url.path == '/buildingSoftwareRelease':
         return {'status_code': 200}
+      if url.path == '/destroyedSoftwareRelease':
+        return {'status_code': 200}
       if url.path == '/softwareReleaseError':
         software.error_log = '\n'.join(
             [
@@ -1921,6 +1923,41 @@ echo %s; echo %s; exit 42""" % (line1, line2))
       self.launchSlapgridSoftware()
     self.assertIn('shared-part-list = %s' % self.shared_parts_root, software.error_log)
 
+  def test_remove_software(self):
+    computer = ComputerForTest(self.software_root, self.instance_root, 1, 1)
+    with httmock.HTTMock(computer.request_handler):
+      software = computer.software_list[0]
+
+      software.setBuildout("""#!/bin/sh
+mkdir directory
+touch directory/file
+""")
+      self.launchSlapgridSoftware()
+      self.assertIn('directory', os.listdir(os.path.join(self.software_root, software.software_hash)))
+
+      software.requested_state = 'destroyed'
+      self.launchSlapgridSoftware()
+      self.assertEqual(os.listdir(self.software_root), [])
+
+  def test_remove_software_chmod(self):
+    # This software is "hard" to remove, as permissions have been changed
+    computer = ComputerForTest(self.software_root, self.instance_root, 1, 1)
+    with httmock.HTTMock(computer.request_handler):
+      software = computer.software_list[0]
+
+      software.setBuildout("""#!/bin/sh
+mkdir directory
+touch directory/file
+chmod a-rxw directory/file
+chmod a-rxw directory
+""")
+      self.launchSlapgridSoftware()
+      self.assertIn('directory', os.listdir(os.path.join(self.software_root, software.software_hash)))
+
+      software.requested_state = 'destroyed'
+      self.launchSlapgridSoftware()
+      self.assertEqual(os.listdir(self.software_root), [])
+
 
 class SlapgridInitialization(unittest.TestCase):
   """

slapos/util.py

@@ -39,6 +39,14 @@ import six
 from six.moves.urllib import parse
 import hashlib
 import netaddr
+import shutil
+
+
+try:
+  PermissionError
+except NameError:  # make pylint happy on python2...
+  PermissionError = Exception
+
 
 
 def mkdir_p(path, mode=0o700):
@@ -221,3 +229,39 @@ def _addIpv6Brackets(url):
       api_netloc = '%s:%s' % (api_netloc, port)
     url = parse.urlunsplit((api_scheme, api_netloc, api_path, api_query, api_fragment))
   return url
+
+
+def rmtree(path):
+  """Delete a path recursively.
+  Like shutil.rmtree, but supporting the case that some files or folder
+  might have been marked read only.  """
+  def chmod_retry(func, failed_path, exc_info):
+    """Make sure the directories are executable and writable.
+    """
+    # Depending on the Python version, the following items differ.
+    if six.PY3:
+      expected_error_type = PermissionError
+      expected_func = os.lstat
+    else:
+      expected_error_type = OSError
+      expected_func = os.listdir
+    e = exc_info[1]
+    if isinstance(e, expected_error_type):
+      if e.errno == errno.ENOENT:
+        # because we are calling again rmtree on listdir errors, this path might
+        # have been already deleted by the recursive call to rmtree.
+        return
+      if e.errno == errno.EACCES:
+        if func is expected_func:
+          os.chmod(failed_path, 0o700)
+          # corner case to handle errors in listing directories.
+          # https://bugs.python.org/issue8523
+          return shutil.rmtree(failed_path, onerror=chmod_retry)
+        # If parent directory is not writable, we still cannot delete the file.
+        # But make sure not to change the parent of the folder we are deleting.
+        if failed_path != path:
+          os.chmod(os.path.dirname(failed_path), 0o700)
+          return func(failed_path)
+    raise e  # XXX make pylint happy
+
+  shutil.rmtree(path, onerror=chmod_retry)