ContributionTool.py 30.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
##############################################################################
#
# Copyright (c) 2007 Nexedi SARL and Contributors. All Rights Reserved.
#                    Jean-Paul Smets <jp@nexedi.com>
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

29
import cStringIO
30 31
import re
import string
32 33
import socket
import md5
Jean-Paul Smets's avatar
Jean-Paul Smets committed
34
import urllib2, urllib
35

Bartek Górny's avatar
Bartek Górny committed
36
from AccessControl import ClassSecurityInfo, getSecurityManager
37 38 39 40
from Globals import InitializeClass, DTMLFile
from Products.ERP5Type.Tool.BaseTool import BaseTool
from Products.ERP5Type import Permissions
from Products.ERP5 import _dtmldir
Jean-Paul Smets's avatar
Jean-Paul Smets committed
41 42
from Products.ERP5.Document.Url import no_crawl_protocol_list, no_host_protocol_list

43 44 45 46
from zLOG import LOG
from DateTime import DateTime
from Acquisition import aq_base

47 48 49 50 51 52 53
# Install openers
import ContributionOpener
opener = urllib2.build_opener(ContributionOpener.DirectoryFileHandler)
urllib2.install_opener(opener)

# A temporary hack until urllib2 supports timeout setting - XXX
import socket
54
socket.setdefaulttimeout(600) # 1 minute timeout
55 56

# Global parameters
57
TEMP_NEW_OBJECT_KEY = '_v_new_object'
58
MAX_REPEAT = 10
59 60

_marker = []  # Create a new marker object.
61 62 63 64

class ContributionTool(BaseTool):
  """
    ContributionTool provides an abstraction layer to unify the contribution
65
    of documents into an ERP5 Site.
66

67 68
    ContributionTool needs to be configured in portal_types (allowed contents) so
    that it can store Text, Spreadsheet, PDF, etc. 
69

70 71 72
    The main method of ContributionTool is newContent. This method can
    be provided various parameters from which the portal type and document
    metadata can be derived. 
73 74

    Configuration Scripts:
Jean-Paul Smets's avatar
Jean-Paul Smets committed
75

76 77 78
      - ContributionTool_getPropertyDictFromFileName: receives file name and a 
        dict derived from filename by regular expression, and does any necesary
        operations (e.g. mapping document type id onto a real portal_type).
Jean-Paul Smets's avatar
Jean-Paul Smets committed
79 80 81 82 83 84

    Problems which are not solved

      - handling of relative links in HTML contents (or others...)
        some text rewriting is necessary.

85 86 87 88 89 90
  """
  title = 'Contribution Tool'
  id = 'portal_contributions'
  meta_type = 'ERP5 Contribution Tool'
  portal_type = 'Contribution Tool'

Jean-Paul Smets's avatar
Jean-Paul Smets committed
91 92 93
  # Regular expressions
  simple_normaliser = re.compile('#.*')

94 95 96 97 98 99 100
  # Declarative Security
  security = ClassSecurityInfo()

  security.declareProtected(Permissions.ManagePortal, 'manage_overview' )
  manage_overview = DTMLFile( 'explainContributionTool', _dtmldir )

  security.declarePrivate('findTypeName')
101
  def findTypeName(self, file_name, document, container=None):
102 103
    """
      Finds the appropriate portal type based on the file name
104
      or if necessary the content of the document.
Jean-Paul Smets's avatar
Jean-Paul Smets committed
105 106 107 108

      NOTE: XXX This implementation can be greatly accelerated by
      caching a dict resulting which combines getContentTypeRegistryTypeDict
      and valid_portal_type_list
109
    """
Jean-Paul Smets's avatar
Jean-Paul Smets committed
110 111 112 113 114 115 116
    def getContentTypeRegistryTypeDict():
      result = {}
      for id, pred in self.content_type_registry.listPredicates():
        (p, type) = pred
        result[type] = None
      return result

117
    portal_type = None
118
    # We should only consider those portal_types which share the
Jean-Paul Smets's avatar
Jean-Paul Smets committed
119 120 121 122 123 124 125 126
    # same constructor with the current object and which are not
    # part of the definitions of content_type_registry. For
    # example if content type registry has a definition for
    # RSS feed, then there is no reason to consider this type
    # whenever receiving some text/html content although both
    # types share the same constructor. However, if Memo has
    # same constructor as Text and Memo is not in content_type_registry
    # then it should be considered.
127
    extra_valid_portal_type_list = []
Jean-Paul Smets's avatar
Jean-Paul Smets committed
128 129 130 131 132
    content_registry_type_dict = getContentTypeRegistryTypeDict()
    portal_type_tool = self.portal_types
    for pt in portal_type_tool.objectValues():
      if hasattr(pt, 'factory') and pt.factory == portal_type_tool[document.getPortalType()].factory:
        if not content_registry_type_dict.has_key(pt.id):
133 134 135 136 137 138 139 140 141 142
          extra_valid_portal_type_list.append(pt.id)

    if not extra_valid_portal_type_list:
      # There is really no ambiguity here
      # The portal_type set by PUT_factory is appropriate
      # This is the best case we can get
      # LOG('findTypeName no ambiguity', 0, document.portal_type)
      return document.portal_type

    valid_portal_type_list = [document.portal_type] + extra_valid_portal_type_list
143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
    # LOG('valid_portal_type_list', 0, str(valid_portal_type_list))

    # If a container is defined, filter valid portal types with allowedContentTypes
    if container is not None:
      allowed_type_list = map(lambda x: x.id, container.allowedContentTypes())
      # LOG('allowed_type_list', 0, str(allowed_type_list))
      valid_portal_type_list = filter(lambda x: x in allowed_type_list, valid_portal_type_list)
      # LOG('filtered valid_portal_type_list', 0, str(valid_portal_type_list))

    # Check if there is any intersection with index portal types
    # If not, we do not need to even check if content is an index
    is_index_candidate = False
    for index_type in self.getPortalCrawlerIndexTypeList():
      if index_type in valid_portal_type_list:
        is_index_candidate = True
        candidate_index_type = index_type

    if is_index_candidate and document.isIndexContent(container=container):
      # If this document has to be created inside an External Source (container)
      # we need to analyse its content to determine whether it is or not
      # an index document. Index documents should not be searchable as documents
      # and should not be considered in the depth calculation of the crawling 
      # process
      return candidate_index_type # We suppose that there is only one index type in allowed content types
167 168

    # Check if the filename tells which portal_type this is
169
    portal_type_list = self.getPropertyDictFromFileName(file_name).get('portal_type', [])
170 171 172 173
    if isinstance(portal_type_list, str): portal_type_list = [portal_type_list]
    portal_type_list = filter(lambda x: x in valid_portal_type_list, portal_type_list)
    if not portal_type_list:
      portal_type_list = valid_portal_type_list
174 175
    if len(portal_type_list) == 1:
      # if we have only one, then this is it
176
      # LOG('findTypeName single portal_type_list', 0, portal_type_list[0])
177
      return portal_type_list[0]
178

179 180
    # If it is still None, we need to read the document
    # to check which of the candidates is suitable
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196
    # Let us give a chance to getPropertyDictFromContent to
    # tell us what is the portal type of this document
    content_portal_type_list = document.getPropertyDictFromContent().get('portal_type', None)
    if content_portal_type_list:
      if isinstance(portal_type, str):
        content_portal_type_list = [content_portal_type_list]
      # Filter valid candidates
      content_portal_type_list = filter(lambda x: x in portal_type_list, content_portal_type_list)
      if content_portal_type_list:
        # if we have more than one, then return the first one
        # LOG('findTypeName from content', 0, content_portal_type_list[0])
        return content_portal_type_list[0]

    # If portal_type_list is not empty, return the first one
    # LOG('findTypeName from first portal_type_list', 0, portal_type_list[0])
    return portal_type_list[0]
197 198

  security.declareProtected(Permissions.AddPortalContent, 'newContent')
199
  def newContent(self, id=None, portal_type=None, url=None, container=None,
Jean-Paul Smets's avatar
Jean-Paul Smets committed
200
                       container_path=None,
201 202 203 204 205 206 207 208 209
                       discover_metadata=1, temp_object=0,
                       user_login=None, **kw):
    """
      The newContent method is overriden to implement smart content
      creation by detecting the portal type based on whatever information
      was provided and finding out the most appropriate module to store
      the content.

      user_login is the name under which the content will be created
210 211
      XXX - this is a security hole which needs to be fixed by
      making sure only Manager can use this parameter
212

213 214 215 216
      container -- if specified, it is possible to define
      where to contribute the content. Else, ContributionTool
      tries to guess.

Jean-Paul Smets's avatar
Jean-Paul Smets committed
217 218 219
      container_path -- if specified, defines the container path
      and has precedence over container

220 221
      url -- if specified, content is download from the URL.

222 223 224 225 226 227 228 229 230 231
      NOTE:
        We always generate ID. So, we must prevent using the one
        which we were provided.
    """
    # Temp objects use the standard newContent from Folder
    if temp_object:
      # For temp_object creation, use the standard method
      return BaseTool.newContent(self, id=id, portal_type=portal_type, temp_object=1, **kw)

    # Try to find the file_name
232
    file_name = None
233
    mime_type = None
234
    if not url:
235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
      # check if file was provided
      file = kw.get('file', None)
      if file is not None:
        file_name = file.filename
      else:
        # some channels supply data and file-name separately
        # this is the case for example for email ingestion
        # in this case, we build a file wrapper for it
        data = kw.get('data', None)
        if data is not None:
          file_name = kw.get('file_name', None)
          if file_name is not None:
            file = cStringIO.StringIO()
            file.write(data)
            file.seek(0)
250 251 252
            kw['file'] = file
            del kw['data']
            del kw['file_name']
Jean-Paul Smets's avatar
Jean-Paul Smets committed
253
    else:
254
      # build a new file from the url
255 256
      url_file = urllib2.urlopen(url)
      data = url_file.read() # time out must be set or ... too long XXX
257 258 259
      file = cStringIO.StringIO()
      file.write(data)
      file.seek(0)
260
      # Create a file name based on the URL and quote it
Jean-Paul Smets's avatar
Jean-Paul Smets committed
261
      file_name = url.split('/')[-1] or url.split('/')[-2]
262 263 264
      file_name = urllib.quote(file_name, safe='')
      file_name = file_name.replace('%', '')
      # For URLs, we want an id by default equal to the encoded URL 
265
      if id is None: id = self.encodeURL(url)
266 267
      if hasattr(url_file, 'headers'):
        headers = url_file.headers
268 269 270
        if hasattr(headers, 'type'):
          mime_type = headers.type
      kw['file'] = file
271 272

    # If the portal_type was provided, we can go faster
273 274 275 276
    if portal_type and container is None:
      # We know the portal_type, let us find the default module
      # and use it as container
      container = self.getDefaultModule(portal_type)
277

278 279
    if portal_type and container is not None:
      # We could simplify things here and return a document immediately
280
      # NOTE: we use the module ID generator rather than the provided ID
281 282 283 284 285
      #document = module.newContent(portal_type=portal_type, **kw)
      #if discover_metadata:
      #  document.activate().discoverMetadata(file_name=file_name, user_login=user_login)
      #return document
      pass # XXX - This needs to be implemented once the rest is stable
286

287
    # From here, there is no hope unless a file was provided
288 289 290 291
    if file is None:
      raise ValueError, "could not determine portal type"

    # So we will simulate WebDAV to get an empty object
Jean-Paul Smets's avatar
Jean-Paul Smets committed
292
    # with PUT_factory - we provide the mime_type as
293
    # parameter
294
    # LOG('new content', 0, "%s -- %s" % (file_name, mime_type))
Jean-Paul Smets's avatar
Jean-Paul Smets committed
295
    ob = self.PUT_factory(file_name, mime_type, None)
296

Jean-Paul Smets's avatar
Jean-Paul Smets committed
297 298 299 300
    # Raise an error if we could not guess the portal type
    if ob is None:
      raise ValueError, "Could not determine the document type"

301 302 303
    # Prevent any reindexing operations
    ob.isIndexable = 0

304
    # Then put the file inside ourselves for a short while
Jean-Paul Smets's avatar
Jean-Paul Smets committed
305
    BaseTool._setObject(self, file_name, ob)
306
    document = BaseTool._getOb(self, file_name)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
307

308 309 310
    try:
      # Then edit the document contents (so that upload can happen)
      document._edit(**kw)
311 312
      if url:
        document.fromURL(url)
313 314 315
    finally:
      # Remove the object from ourselves
      BaseTool._delObject(self, file_name)
316

Jean-Paul Smets's avatar
Jean-Paul Smets committed
317
    # Move the document to where it belongs
Jean-Paul Smets's avatar
Jean-Paul Smets committed
318 319
    if container_path is not None:
      container = self.getPortalObject().restrictedTraverse(container_path)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
320 321 322
    document = self._setObject(file_name, ob, user_login=user_login, id=id,
                               container=container, discover_metadata=discover_metadata,
                               )
323
    document = self._getOb(file_name) # Call _getOb to purge cache
Jean-Paul Smets's avatar
Jean-Paul Smets committed
324

Jean-Paul Smets's avatar
Jean-Paul Smets committed
325
    # Notify workflows
326
    #document.notifyWorkflowCreated()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
327

328 329
    # Allow reindexing, reindex it and return the document
    delattr(document, 'isIndexable')
330
    document.reindexObject()
331 332
    return document

333
  security.declareProtected( Permissions.AddPortalContent, 'newXML' )
334 335 336 337 338 339 340 341
  def newXML(self, xml):
    """
      Create a new content based on XML data. This is intended for contributing
      to ERP5 from another application.
    """
    pass

  security.declareProtected(Permissions.ModifyPortalContent,'getPropertyDictFromFileName')
342
  def getPropertyDictFromFileName(self, file_name):
343 344 345 346
    """
      Gets properties from filename. File name is parsed with a regular expression
      set in preferences. The regexp should contain named groups.
    """
347 348 349
    if file_name is None:
      return {}
    property_dict = {}
350
    rx_src = self.portal_preferences.getPreferredDocumentFileNameRegularExpression()
351 352 353
    if rx_src in ('', None):
      # we must have file name regular expression defined in preferences
      raise ValueError, '[DMS] No file name regular expression defined in preferences.'
354
    if rx_src:
355
      rx_parse = re.compile(rx_src)
356 357 358 359 360
      if rx_parse is not None:
        try:
          property_dict = rx_parse.match(file_name).groupdict()
        except AttributeError: # no match
          pass
361 362
    method = self._getTypeBasedMethod('getPropertyDictFromFileName', 
        fallback_script_id = 'ContributionTool_getPropertyDictFromFileName')
363
    property_dict = method(file_name, property_dict)
Ivan Tyagov's avatar
Ivan Tyagov committed
364
    if property_dict.get('portal_type', None) is not None:
365
      # we have to return portal_type as a tuple
366
      # because we should allow for having multiple candidate types
367 368
      property_dict['portal_type'] = (property_dict['portal_type'],)
    else:
369
      # we have to find candidates by file extenstion
Ivan Tyagov's avatar
Ivan Tyagov committed
370 371 372
      if file_name.rfind('.')!= -1:
        ext = file_name.split('.')[-1]
        property_dict['portal_type'] = self.ContributionTool_getCandidateTypeListByExtension(ext)
373 374
    return property_dict

375
  # WebDAV virtual folder support
Jean-Paul Smets's avatar
Jean-Paul Smets committed
376 377
  def _setObject(self, name, ob, user_login=None, container=None,
                       id=None, discover_metadata=1):
378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398
    """
      The strategy is to let NullResource.PUT do everything as
      usual and at the last minute put the object in a different
      location with a different portal type. This means that
      NullResource.PUT creates an empty document with PUT_factory
      then upload document data by invoking PUT on the empty
      document and finally sets the object. By overriding _setObject
      we get a chance to fix the portal_type of the document
      (as long as the one we find is compatible) and move the
      document to the appropriate module.

      content_type_registry must be set up so that an appropriate
      portal_type with appropriate meta_type is found for every
      kind of document. However, a different portal_type might
      be used in the end.

      The ContributionTool instance must be configured in such
      way that _verifyObjectPaste will return TRUE.

      Refer to: NullResource.PUT
    """
399 400 401 402 403 404 405 406 407 408 409 410 411
    # _setObject is called by constructInstance at a time
    # when the object has no portal_type defined yet. It
    # will be removed later on. We can safely store the
    # document inside us at this stage. Else we
    # must find out where to store it.
    if not ob.__dict__.has_key('portal_type'):
      BaseTool._setObject(self, name, ob)
      document = self[name]
    else:
      # We give the system a last chance to analyse the
      # portal_type based on the document content
      # (ex. a Memo is a kind of Text which can be identified
      # by the fact it includes some specific content)
412
      portal_type = self.findTypeName(name, ob.__of__(self), container=container)
413 414 415 416 417 418 419 420
      if portal_type is None: portal_type = ob.portal_type
      ob._setPortalTypeName(portal_type) # This is redundant with finishConstruction
                                       # but necessary to move objects to appropriate
                                       # location based on their content. Since the
                                       # object is already constructed here, we
                                       # can safely change its portal_type
      # Now we know the portal_type, let us find the module
      # to which we should move the document to
421 422 423 424
      if container is None:
        module = self.getDefaultModule(ob.portal_type)
      else:
        module = container
Jean-Paul Smets's avatar
Jean-Paul Smets committed
425 426 427 428
      if id is None:
        new_id = module.generateNewId()
      else:
        new_id = id
429
      ob.id = new_id
430 431 432 433 434 435 436 437 438 439 440 441 442 443 444
      existing_document = module.get(new_id, None)
      if existing_document is None:
        # There is no preexisting document - we can therefore
        # set the new object
        module._setObject(new_id, ob)
        # We can now discover metadata
        document = module[new_id]
        if discover_metadata:
          # Metadata disovery is done as an activity by default
          # If we need to discoverMetadata synchronously, it must
          # be for user interface and should thus be handled by
          # ZODB scripts
          document.activate().discoverMetadata(file_name=name, user_login=user_login)
      else:
        if document.isExternalDocument():
445
          document = existing_document
446
          # If this is an external document, update its content
447
          # document.activate().updateContentFromURL() # XXX I think this is no longer useful with alarms
Jean-Paul Smets's avatar
Jean-Paul Smets committed
448 449 450 451
          # XXX - Make sure this does not increase ZODB
          # XXX - what to do also with parameters (put again edit_kw) ?
          # Providing some information to the use about the fact
          # this was an existing document would also be great
452
        else:
Jean-Paul Smets's avatar
Jean-Paul Smets committed
453 454
          # We may have to implement additional revision support
          # to support in place contribution (ie. for a given ID)
455
          # but is this really useful ?
456
          raise NotImplementedError
457

458 459
      # Keep the document close to us - this is only useful for
      # file upload from webdav
460 461 462
      if not hasattr(self, '_v_document_cache'):
        self._v_document_cache = {}
      self._v_document_cache[name] = document.getRelativeUrl()
463 464 465

    # Return document to newContent method
    return document
466

467 468 469 470 471
  def _getOb(self, id, default=_marker):
    """
    Check for volatile temp object info first
    and try to find it
    """
472 473
    # Use the document cache if possible and return result immediately
    # this is only useful for webdav
Jean-Paul Smets's avatar
Jean-Paul Smets committed
474 475 476
    if hasattr(self, '_v_document_cache'):
      document_url = self._v_document_cache.get(id, None)
      if document_url is not None:
477
        del self._v_document_cache[id]
Jean-Paul Smets's avatar
Jean-Paul Smets committed
478 479
        return self.getPortalObject().unrestrictedTraverse(document_url)

480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496
    # Try first to return the real object inside
    # This is much safer than trying to access objects displayed by listDAVObjects
    # because the behaviour of catalog is unpredicatble if a string is passed
    # for a UID. For example 
    #   select path from catalog where uid = "001193.html";
    # will return the same as
    #   select path from catalog where uid = 1193;
    # This was the source of an error in which the contribution tool
    # was creating a web page and was returning a Base Category
    # when
    #   o = folder._getOb(id)
    # was called in DocumentConstructor
    result = BaseTool._getOb(self, id, default=default)
    if result is not _marker:
      return result

    # Return an object listed by listDAVObjects
497 498 499
    uid = str(id).split('-')[-1]
    object = self.getPortalObject().portal_catalog.unrestrictedGetResultValue(uid=uid)
    if object is not None:
500
      return object.getObject() # Make sure this does not break security. XXX
501

502 503 504
    # Raise an AttributeError the same way as in OFS.ObjectManager._getOb
    raise AttributeError, id

505

Bartek Górny's avatar
Bartek Górny committed
506
  def listDAVObjects(self):
507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530
    """
      Get all contents contributed by the current user. This is
      delegated to a script in order to help customisation.
    """
    method = getattr(self, 'ContributionTool_getMyContentList', None)
    if method is not None:
      object_list = method()
    else:
      sm = getSecurityManager()
      user = sm.getUser()
      object_list = self.portal_catalog(portal_type=self.getPortalMyDocumentTypeList(),
                                        owner=str(user))

    def wrapper(o_list):
      for o in o_list:
        o = o.getObject()
        reference = o.getReference()
        if reference:
          id = '%s-%s' % (reference, o.getUid())
        else:
          id = '%s' % o.getUid()
        yield o.getObject().asContext(id=id)

    return wrapper(object_list)
Bartek Górny's avatar
Bartek Górny committed
531

Jean-Paul Smets's avatar
Jean-Paul Smets committed
532
  # Crawling methods
533 534
  security.declareProtected(Permissions.View, 'normaliseURL')
  def normaliseURL(self, url, base_url=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552
    """
      Returns a normalised version of the url so
      that we do not download twice the same content.
      URL normalisation is an important part in crawlers.
      The current implementation is obviously simplistic.
      Refer to http://en.wikipedia.org/wiki/Web_crawler
      and study Harvestman for more ideas.
    """
    url = self.simple_normaliser.sub('', url)
    url_split = url.split(':')
    url_protocol = url_split[0]
    if url_protocol in no_host_protocol_list:
      return url
    if base_url and len(url_split) == 1:
      # Make relative URL absolute
      url = '%s/%s' % (base_url, url)
    return url

553 554
  security.declareProtected(Permissions.View, 'encodeURL')
  def encodeURL(self, url):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
555 556 557 558 559 560 561 562 563 564
    """
    Returns the URL as an ID. ID should be chosen in such
    way that it is optimal with HBTreeFolder (ie. so that
    distribution of access time on a cluster is possible)

    NOTE: alternate approach is based on a url table
    and catalog lookup. It is faster ? Not sure. Since
    we must anyway insert objects in btrees and this
    is simimar in cost to accessing them.
    """
565 566 567
    # Produce an MD5 from the URL
    hex_md5 = md5.md5(url).hexdigest()
    # Take the first part in the URL which is not empty
568
    # LOG("encodeURL", 0, url)
569 570 571 572 573 574 575 576 577 578 579 580 581
    url_segment = url.split(':')[1]
    url_segment_list = url_segment.split('/')
    url_domain = None
    for url_part in url_segment_list:
      if url_part:
        url_domain = url_part
        break
    # Return encoded url
    if url_domain:
      url_domain = urllib.quote(url_domain, safe='')
      url_domain = url_domain.replace('%', '')
      return "%s-%s" % (url_domain, hex_md5)
    return hex_md5
Jean-Paul Smets's avatar
Jean-Paul Smets committed
582 583 584 585 586 587
    url = urllib.quote(url, safe='')
    url = url.replace('_', '__')
    url = url.replace('%', '_')
    return url

  security.declareProtected(Permissions.AddPortalContent, 'crawlContent')
588
  def crawlContent(self, content, container=None):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
589 590 591 592 593 594 595
    """
      Analyses content and download linked pages

      XXX: missing is the conversion of content local href to something
      valid.
    """
    depth = content.getCrawlingDepth()
596 597 598 599 600 601 602 603
    if depth < 0:
      # Do nothing if crawling depth is reached
      # (this is not a duplicate code but a way to prevent
      # calling isIndexContent unnecessarily)
      return
    if not content.isIndexContent(): # Decrement depth only if it is a content document
      depth = depth - 1
    if depth < 0:
Jean-Paul Smets's avatar
Jean-Paul Smets committed
604 605 606
      # Do nothing if crawling depth is reached
      return
    base_url = content.getContentBaseURL()
607
    url_list = map(lambda url: self.normaliseURL(url, base_url), set(content.getContentURLList()))
Jean-Paul Smets's avatar
Jean-Paul Smets committed
608
    for url in set(url_list):
609
      # LOG('trying to crawl', 0, url)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
610 611 612
      # Some url protocols should not be crawled
      if url.split(':')[0] in no_crawl_protocol_list:
        continue
613 614 615 616
      if container is None:
        #if content.getParentValue()
        # in place of not ?
        container = content.getParentValue()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
617
      # Calculate the id under which content will be stored
618
      id = self.encodeURL(url)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
619 620 621 622 623 624
      # Try to access the document if it already exists
      document = container.get(id, None)
      if document is None:
        # XXX - This call is not working due to missing group_method_id
        # therefore, multiple call happen in parallel and eventually fail
        # (the same URL is created multiple times)
625
        # LOG('activate newContentFromURL', 0, url)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
626
        self.activate(activity="SQLQueue").newContentFromURL(container_path=container.getRelativeUrl(),
627 628 629 630 631
                                                      id=id, url=url, crawling_depth=depth)
      elif depth and document.getCrawlingDepth() < depth:
        # Update the crawling depth if necessary
        document._setCrawlingDepth(depth)
        document.activate().crawlContent()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
632 633

  security.declareProtected(Permissions.AddPortalContent, 'updateContentFromURL')
634
  def updateContentFromURL(self, content, repeat=MAX_REPEAT, crawling_depth=0):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
635 636 637
    """
      Updates an existing content.
    """
638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676
    # First, test if the document is updatable according to
    # its workflow states (if it has a workflow associated with)
    if content.isUpdatable():
      # Step 0: update crawling_depth if required
      if crawling_depth > content.getCrawlingDepth():
        content._setCrawlingDepth(crawling_depth)
      # Step 1: download new content
      try:
        url = content.asURL()
        data = urllib2.urlopen(url).read()
        file = cStringIO.StringIO()
        file.write(data)
        file.seek(0)
      except urllib2.HTTPError, error:
        if repeat == 0:
          # XXX - Call the extendBadURLList method,--NOT Implemented--
          # IDEA : ajouter l'url en question dans une list "bad_url_list"  puis lors du crawling au lieu que de boucler sur 
          #        la liste des url extraites de la page web on fait un test supplementaire qui verifie que l'url n'est pas 
          #        dans la liste  bad_url_lis
          raise urllib2.HTTPError
        content.activate(at_date=DateTime() + 1).updateContentFromURL(repeat=repeat - 1)
        return
      except urllib2.URLError, error:
        if repeat == 0:
          # XXX - Call the extendBadURLList method,--NOT Implemented--
          raise urllib2.URLError
        content.activate(at_date=DateTime() + 1).updateContentFromURL(repeat=repeat - 1)
        return

      # Step 2: compare and update if necessary (md5)
      # md5 stuff to compare contents
      new_content_md5 = md5.md5(data).hexdigest()
      content_md5 = content.getContentMd5()
      if content_md5 is new_content_md5:
        return
      content._edit(file=file)# Please make sure that if content is the same
                              # we do not update it
                              # This feature must be implemented by Base or File
                              # not here (look at _edit in Base)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
677
      # Step 3: convert to base format
678
      content.convertToBaseFormat()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
679
      # Step 4: activate populate (unless interaction workflow does it)
680
      content.activate().populateContent()
Jean-Paul Smets's avatar
Jean-Paul Smets committed
681
      # Step 5: activate crawlContent
682 683 684 685
      depth = content.getCrawlingDepth()
      if depth > 0:
        content.activate().crawlContent()
      content.setContentMd5(new_content_md5)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
686 687

  security.declareProtected(Permissions.AddPortalContent, 'newContentFromURL')
688
  def newContentFromURL(self, container_path=None, id=None, repeat=MAX_REPEAT, **kw):
Jean-Paul Smets's avatar
Jean-Paul Smets committed
689 690 691 692 693 694 695 696 697 698
    """
      A wrapper method for newContent which provides extra safety
      in case or errors (ie. download, access, conflict, etc.).
      The method is able to handle a certain number of exceptions
      and can postpone itself through an activity based on
      the type of exception (ex. for a 404, postpone 1 day), using
      the at_date parameter and some standard values.

      NOTE: implementation needs to be done.
    """
699 700 701 702 703 704 705 706 707
    # First of all, make sure do not try to create an existing document
    if container_path is not None and id is not None:
      container = self.restrictedTraverse(container_path)
      document = container.get(id, None)
      if document is not None:
        # Document aleardy exists: no need to keep on crawling
        return
    try:
      document = self.newContent(container_path=container_path, id=id, **kw)
708 709 710 711 712 713
      if document.isIndexContent() and document.getCrawlingDepth() >= 0:
        # If this is an index document, keep on crawling even if crawling_depth is 0
        document.activate().crawlContent()
      elif document.getCrawlingDepth() > 0:
        # If this is an index document, stop crawling if crawling_depth is 0
        document.activate().crawlContent()
714
    except urllib2.HTTPError, error:
715 716 717 718 719
      if repeat == 0:
        # here we must call the extendBadURLList method,--NOT Implemented--
        # which had to add this url to bad URL list, so next time we avoid
        # crawling bad URL
        raise urllib2.HTTPError
720 721 722 723 724
      # Catch any HTTP error
      self.activate(at_date=DateTime() + 1).newContentFromURL(
                        container_path=container_path, id=id,
                        repeat=repeat - 1, **kw)
    except urllib2.URLError, error:
725 726 727 728 729 730 731 732 733 734 735
      if repeat == 0:
        # XXX - Call the extendBadURLList method, --NOT Implemented--
        raise urllib2.URLError
      print error.reason
      #if getattr(error.reason,'args',None):
        #if error.reason.args[0] == socket.EAI_AGAIN:
          ## Temporary failure in name resolution - try again in 1 day
      self.activate(at_date=DateTime() + 1,
                    activity="SQLQueue").newContentFromURL(
                      container_path=container_path, id=id,
                      repeat=repeat - 1, **kw)
Jean-Paul Smets's avatar
Jean-Paul Smets committed
736

Ivan Tyagov's avatar
Ivan Tyagov committed
737
InitializeClass(ContributionTool)