utils.py 11.6 KB
Newer Older
Łukasz Nowak's avatar
Łukasz Nowak committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################
import logging
import hashlib
import os
30
import pkg_resources
Łukasz Nowak's avatar
Łukasz Nowak committed
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
import stat
import subprocess
import sys
import pwd
import grp
from exception import BuildoutFailedError, WrongPermissionError
from hashlib import md5

# Such umask by default will create paths with full permission
# for user, non writable by group and not accessible by others
SAFE_UMASK = 027

PYTHON_ENVIRONMENT_REMOVE_LIST = [
  'PYTHONHOME',
  'PYTHONPATH',
  'PYTHONSTARTUP',
  'PYTHONY2K',
  'PYTHONOPTIMIZE',
  'PYTHONDEBUG',
  'PYTHONDONTWRITEBYTECODE',
  'PYTHONINSPECT',
  'PYTHONNOUSERSITE',
  'PYTHONNOUSERSITE',
  'PYTHONUNBUFFERED',
  'PYTHONVERBOSE',
]

SYSTEM_ENVIRONMENT_REMOVE_LIST = [
  'ENV',
  'LOGNAME',
  'TEMP',
  'TMP',
  'TMPDIR',
  'USER',
]

LOCALE_ENVIRONEMNT_REMOVE_LIST = [
  'LANG',
  'LANGUAGE',
  'LC_ADDRESS',
  'LC_COLLATE',
  'LC_CTYPE',
  'LC_IDENTIFICATION',
  'LC_MEASUREMENT',
  'LC_MESSAGES',
  'LC_MONETARY',
  'LC_NAME',
  'LC_NUMERIC',
  'LC_PAPER',
  'LC_SOURCED',
  'LC_TELEPHONE',
  'LC_TIME',
]

class AlreadyRunning(Exception):
  pass


class SlapPopen(subprocess.Popen):
  """Almost normal subprocess with gridish features"""
  def __init__(self, *args, **kwargs):
    kwargs.update(stdin=subprocess.PIPE)
    subprocess.Popen.__init__(self, *args, **kwargs)
    self.stdin.flush()
    self.stdin.close()
    self.stdin = None


def getSoftwareUrlHash(url):
  return md5(url).hexdigest()


def getCleanEnvironment(home_path='/tmp'):
  logger = logging.getLogger('CleanEnvironment')
  changed_env = {}
  removed_env = []
  env = os.environ.copy()
  # Clean python related environment variables
  for k in PYTHON_ENVIRONMENT_REMOVE_LIST + SYSTEM_ENVIRONMENT_REMOVE_LIST \
      + LOCALE_ENVIRONEMNT_REMOVE_LIST:
    old = env.pop(k, None)
    if old is not None:
      removed_env.append(k)
  changed_env['HOME'] = env['HOME'] = home_path
  for k in sorted(changed_env.iterkeys()):
    logger.debug('Overriden %s = %r' % (k,changed_env[k]))
  logger.debug('Removed from environement: %s' % ', '.join(sorted(removed_env)))
  return env


def setRunning(pid_file):
  """Creates a pidfile. If a pidfile already exists, we exit"""
  logger = logging.getLogger('Slapgrid')
  if os.path.exists(pid_file):
    # Pid file is present
    logger.warning('pid file already exists : %s' % (pid_file))
    try:
      pid = int(open(pid_file, 'r').readline())
    except ValueError:
      pid = None
    # XXX This could use psutil library.
    if pid is not None and os.path.exists("/proc/%s" % pid):
      #XXX: can we trust sys.argv?
      process_name = os.path.basename(sys.argv[0])
      if process_name in open('/proc/%s/cmdline' % pid, 'r').readline():
        # In case process is present, ignore.
        raise AlreadyRunning('A slapgrid process is running with pid %s' % pid)
    logger.info('Pid file %r was stale one, overwritten' % pid_file)
  # Start new process
  write_pid(pid_file)


def setFinished(pid_file):
  try:
    os.remove(pid_file)
  except OSError:
    pass


def write_pid(pid_file):
  logger = logging.getLogger('Slapgrid')
  pid = os.getpid()
  try:
    f = open(pid_file, 'w')
    f.write('%s' % pid)
    f.close()
  except (IOError, OSError):
    logger.critical('slapgrid could not write pidfile %s' % pid_file)
    raise


def dropPrivileges(uid, gid):
  """Drop privileges to uid, gid if current uid is 0

  Do tests to check if dropping was successful and that no system call is able
  to re-raise dropped privileges

  Does nothing in case if uid and gid are not 0
  """
  logger = logging.getLogger('dropPrivileges')
  current_uid, current_gid = os.getuid(), os.getgid()
  if uid == 0 or gid == 0:
    raise OSError('Dropping privileges to uid = %r or ' \
                                      'gid = %r is too dangerous' % (uid, gid))
  if not(current_uid == 0 and current_gid == 0):
    logger.debug('Running as uid = %r, gid = %r, dropping not needed and not '
        'possible' % (current_uid, current_gid))
    return
  # drop privileges
  user_name = pwd.getpwuid(uid)[0]
  group_list = [x.gr_gid for x in grp.getgrall() if user_name in x.gr_mem]
  group_list.append(gid)
  os.initgroups(pwd.getpwuid(uid)[0], gid)
  os.setgid(gid)
  os.setuid(uid)

  # assert that privileges are dropped
  message_pre = 'After dropping to uid = %r and gid = %r ' \
                'and group_list = %s' % (
                            uid, gid, group_list)
  new_uid, new_gid, new_group_list = os.getuid(), os.getgid(), os.getgroups()
  if not (new_uid == uid and new_gid == gid and new_group_list == group_list):
    raise OSError('%s new_uid = %r and new_gid = %r and ' \
                                      'new_group_list = %r which is fatal.'
                                      % (message_pre,
                                         new_uid,
                                         new_gid,
                                         new_group_list))

  # assert that it is not possible to go back to running one
  try:
    try:
      os.setuid(current_uid)
    except OSError:
      try:
        os.setgid(current_gid)
      except OSError:
        try:
          os.setgroups([current_gid])
        except OSError:
          raise
  except OSError:
    pass
  else:
    raise ValueError('%s it was possible to go back to uid = %r and gid = '
        '%r which is fatal.' % message_pre, current_uid, current_gid)
  logger.info('Succesfully dropped privileges to uid=%r gid=%r' % (uid, gid))


220 221
def bootstrapBuildout(path, buildout=None,
    additional_buildout_parametr_list=None, console=False):
Łukasz Nowak's avatar
Łukasz Nowak committed
222 223 224 225 226 227 228 229 230
  if additional_buildout_parametr_list is None:
    additional_buildout_parametr_list = []
  logger = logging.getLogger('BuildoutManager')
  # Reads uid/gid of path, launches buildout with thoses privileges
  stat_info = os.stat(path)
  uid = stat_info.st_uid
  gid = stat_info.st_gid

  invocation_list = [sys.executable, '-S']
231 232 233 234 235 236 237
  if buildout is not None:
    invocation_list.append(buildout)
  else:
    logger.warning('Using old style bootstrap of included bootstrap file. '
      'Consider setting buildout binary location.')
    invocation_list.append(pkg_resources.resource_filename(__name__,
      'zc.buildout-bootstap.py'))
Łukasz Nowak's avatar
Łukasz Nowak committed
238
  invocation_list.extend(additional_buildout_parametr_list)
239
  invocation_list.append('bootstrap')
Łukasz Nowak's avatar
Łukasz Nowak committed
240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
  try:
    umask = os.umask(SAFE_UMASK)
    logger.debug('Set umask from %03o to %03o' % (umask, SAFE_UMASK))
    logger.debug('Invoking: %r in directory %r' % (' '.join(invocation_list),
      path))
    kw = dict()
    if not console:
      kw.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    process_handler = SlapPopen(invocation_list,
            preexec_fn=lambda: dropPrivileges(uid, gid),
            cwd=path, env=getCleanEnvironment(pwd.getpwuid(uid).pw_dir),
            **kw
            )
    result = process_handler.communicate()[0]
    if console:
      result = 'Please consult messages above'
    if process_handler.returncode is None or process_handler.returncode != 0:
      message = 'Failed to run buildout profile in directory %r:\n%s\n' % (
          path, result)
      raise BuildoutFailedError(message)
    else:
      logger.debug('Successful run:\n%s' % result)
  except OSError as error:
    raise BuildoutFailedError(error)
  finally:
    old_umask = os.umask(umask)
    logger.debug('Restore umask from %03o to %03o' % (old_umask, umask))


def launchBuildout(path, buildout_binary,
                   additional_buildout_parametr_list=None, console=False):
  """ Launches buildout."""
  logger = logging.getLogger('BuildoutManager')
  if additional_buildout_parametr_list is None:
    additional_buildout_parametr_list = []
  # Reads uid/gid of path, launches buildout with thoses privileges
  stat_info = os.stat(path)
  uid = stat_info.st_uid
  gid = stat_info.st_gid
  # Extract python binary to prevent shebang size limit
  file = open(buildout_binary, 'r')
  line = file.readline()
  file.close()
  invocation_list = []
  if line.startswith('#!'):
    line = line[2:]
    # Prepares parameters for buildout
    invocation_list = line.split() + [buildout_binary]
  # Run buildout without reading user defaults
  invocation_list.append('-U')
  invocation_list.extend(additional_buildout_parametr_list)
  try:
    umask = os.umask(SAFE_UMASK)
    logger.debug('Set umask from %03o to %03o' % (umask, SAFE_UMASK))
    logger.debug('Invoking: %r in directory %r' % (' '.join(invocation_list),
      path))
    kw = dict()
    if not console:
      kw.update(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    process_handler = SlapPopen(invocation_list,
            preexec_fn=lambda: dropPrivileges(uid, gid), cwd=path,
            env=getCleanEnvironment(pwd.getpwuid(uid).pw_dir), **kw)
    result = process_handler.communicate()[0]
    if console:
      result = 'Please consult messages above'
    if process_handler.returncode is None or process_handler.returncode != 0:
      message = 'Failed to run buildout profile in directory %r:\n%s\n' % (
          path, result)
      raise BuildoutFailedError(message)
    else:
      logger.debug('Successful run:\n%s' % result)
  except OSError as error:
    raise BuildoutFailedError(error)
  finally:
    old_umask = os.umask(umask)
    logger.debug('Restore umask from %03o to %03o' % (old_umask, umask))


def updateFile(file_path, content, mode='0600'):
  """Creates an executable with "content" as content."""
  altered = False
  if not (os.path.isfile(file_path)) or \
    not(hashlib.md5(open(file_path).read()).digest() ==\
        hashlib.md5(content).digest()):
      altered = True
      file_file = open(file_path, 'w')
      file_file.write(content)
      file_file.flush()
      file_file.close()
  os.chmod(file_path, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
  if oct(stat.S_IMODE(os.stat(file_path).st_mode)) != mode:
    os.chmod(file_path, int(mode, 8))
    altered = True
  return altered


def updateExecutable(executable_path, content):
  """Creates an executable with "content" as content."""
  return updateFile(executable_path, content, '0700')


def createPrivateDirectory(path):
  """Creates directory belonging to root with umask 077"""
  if not os.path.isdir(path):
    os.mkdir(path)
  os.chmod(path, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
  permission = oct(stat.S_IMODE(os.stat(path).st_mode))
  if permission not in ('0700'):
    raise WrongPermissionError('Wrong permissions in %s ' \
                                        ': is %s, should be 0700'
                                        % path, permission)