Initial import of 'MimetypesRegistry' and 'PortalTransforms' products.

git-svn-id: https://svn.erp5.org/repos/public/erp5/trunk@16694 20353a03-c40f-0410-a6d1-a30d3c3de9de

product/MimetypesRegistry/ChangeLog

+DONT USE ChangeLog use HISTORY.txt instead.
+
+ 2004-07-24  Christian Heimes <heimes@faho.rwth-aachen.de>
+
+    * Changed version to stick to Archetypes version.

product/MimetypesRegistry/Extensions/Install.py

+import os
+from Products.CMFCore.DirectoryView import addDirectoryViews, registerDirectory, \
+     createDirectoryView, manage_listAvailableDirectories
+from Products.CMFCore.utils import getToolByName, minimalpath
+from Globals import package_home
+from OFS.ObjectManager import BadRequestException
+
+from Products.MimetypesRegistry import GLOBALS, skins_dir
+from Products.MimetypesRegistry.interfaces import IMimetypesRegistry
+from Acquisition import aq_base
+from StringIO import StringIO
+
+def install(self):
+    out = StringIO()
+    id = 'mimetypes_registry'
+    if hasattr(aq_base(self), id):
+        mtr = getattr(self, id)
+        if not IMimetypesRegistry.isImplementedBy(mtr) or \
+          not getattr(aq_base(mtr), '_new_style_mtr', None) == 1:
+            print >>out, 'Removing old mimetypes registry tool'
+            self.manage_delObjects([id,])
+    if not hasattr(self, id):
+        addTool = self.manage_addProduct['MimetypesRegistry'].manage_addTool
+        addTool('MimeTypes Registry')
+        print >>out, 'Installing mimetypes registry tool'
+
+    skinstool=getToolByName(self, 'portal_skins')
+
+    fullProductSkinsPath = os.path.join(package_home(GLOBALS), skins_dir)
+    productSkinsPath = minimalpath(fullProductSkinsPath)
+    registered_directories = manage_listAvailableDirectories()
+    if productSkinsPath not in registered_directories:
+        registerDirectory(skins_dir, GLOBALS)
+    try:
+        addDirectoryViews(skinstool, skins_dir, GLOBALS)
+    except BadRequestException, e:
+        pass  # directory view has already been added
+
+    files = os.listdir(fullProductSkinsPath)
+    for productSkinName in files:
+        if os.path.isdir(os.path.join(fullProductSkinsPath, productSkinName)) \
+               and productSkinName != 'CVS':
+            for skinName in skinstool.getSkinSelections():
+                path = skinstool.getSkinPath(skinName)
+                path = [i.strip() for i in  path.split(',')]
+                try:
+                    if productSkinName not in path:
+                        path.insert(path.index('custom') +1, productSkinName)
+                except ValueError:
+                    if productSkinName not in path:
+                        path.append(productSkinName)
+                path = ','.join(path)
+                skinstool.addSkinSelection(skinName, path)
+
+    return out.getvalue()
+
+def fixUpSMIGlobs(self):
+    from Products.MimetypesRegistry.mime_types import smi_mimetypes
+    from Products.Archetypes.debug import log
+    mtr = getToolByName(self, 'mimetypes_registry')
+    smi_mimetypes.initialize(mtr)
+
+    # Now comes the fun part. For every glob, lookup a extension
+    # matching the glob and unregister it.
+    for glob in mtr.globs.keys():
+        if mtr.extensions.has_key(glob):
+            log('Found glob %s in extensions registry, removing.' % glob)
+            mti = mtr.extensions[glob]
+            del mtr.extensions[glob]
+            if glob in mti.extensions:
+                log('Found glob %s in mimetype %s extensions, '
+                    'removing.' % (glob, mti))
+                exts = list(mti.extensions)
+                exts.remove(glob)
+                mti.extensions = tuple(exts)
+                mtr.register(mti)

product/MimetypesRegistry/Extensions/__init__.py

+#poof, you're a product

product/MimetypesRegistry/HISTORY.txt

+1.4.0-final - 2006-06-16
+========================
+
+  * Use zope.contenttype in favor of zope.app.content_types if available.
+    [hannosch]
+
+1.4.0-beta2 - 2006-05-12
+========================
+
+  * Use zope.app.content_types in favor of OFS.content_types if available.
+    [stefan]
+
+  * Spring-cleaning of tests infrastructure.
+    [hannosch]
+
+1.4.0-beta1 - 2006-03-26
+========================
+
+  * fixed Plone #5027: MimeTypeRegistry.classify doesn't handle 
+    "no mimetype" gracefully. Returns 'None' now.
+    [jensens]
+
+  * fixed http://dev.plone.org/archetypes/ticket/622
+    [jensens] 
+
+1.4.0-alpha02 - 2006-02-23
+==========================
+
+  * ensured that the key gotten back from windows_mimetypes.py existed
+    mark says the best way is to examine each key to ensure its valid but
+    would be slower.
+    [runyaga]
+
+  * removed odd archetypes 1.3 style version checking
+    [jensens]
+
+  * Removed BBB code for CMFCorePermissions import location.
+    [hannosch]
+
+  * removed deprecation warning for ToolInit.
+    [jensens]
+
+  * skip backward compatibility to the times where MTR where part of 
+    PortalTransforms.
+    [jensens]
+
+1.3.8-final02 - 2006-01-15
+==========================
+
+  * nothing - the odd version checking needs a version change to stick to
+    Archetypes version again.
+    [yenzenz]
+
+1.3.8-RC1 - 2005-12-29
+======================
+
+  * Split yet another part of register() into a separate
+    method. Cleanup smi_mimetypes initialize a little bit to to use
+    the new method when adding new mimetypes to a already-registered
+    entry.
+    [dreamcatcher]
+
+  * Include aliases in the list of mimetypes for a entry. Based on
+    patch by Jean Jordaan
+    [dreamcatcher]
+
+  * Use a SAX-based parser instead of minidom to improve Zope startup
+    time (by 17 seconds on my Pismo) and memory footprint.
+    [dreamcatcher]
+
+  * Augment known mimetypes with Windows mimetypes, if available.
+    [dreamcatcher]
+
+1.3.7-final01 - 2005-10-11
+==========================
+
+  * For the sake of sanity, include a 'mime.types' with
+    MimetypesRegistry to minimize the platform-specific differences in
+    mime detection when the python 'mimetypes' module is involved.
+    [dreamcatcher]
+
+  * globs from freedesktop.org shared-mime-info were incorrectly
+    mapped to 'extensions' and never really worked because the code
+    tried to strip a leading dot, where the globs normally had '*.'.
+
+    The side-effect of this is that in *nix, the Python 'mimetypes'
+    module would happily read '/etc/mime.types' and gracefully work
+    (/etc/mime.types has most of the extensions of shared-mime-info
+    but a few), where on Windows it would fail to detect mimetypes by
+    extension.
+    [dreamcatcher]
+
+  * Added support for real globs, using fnmatch.translate and
+    re.compile and a migration function that will be run from Plone
+    2.1.1 migration, with some tests specific for globs read from
+    shared-mime-info.
+    [dreamcatcher]
+
+1.3.6-final01 - 2005-08-30
+==========================
+
+  * after one night sleeping over it I removed the yesterday added method.
+    therefore I added according to some heuristics and OOo-Documentation
+    some magic bytes to magic.py and made better tests.
+    [yenzenz]
+
+  * added a method to detect mimetypes of zipped files,
+    here specialy for OOo now all Openofice files and zip
+    files are detected properly.  my simple tests are working:
+    a OOo-Writer and a simpe zipfile are detected.
+    [yenzenz]
+
+  * updated freedesktop.org.xml file to latest CVS version rev 1.57 from
+    http://cvs.freedesktop.org/mime/shared-mime-info/freedesktop.org.xml
+    [yenzenz]
+
+1.3.5-final03 - 2005-08-07
+==========================
+
+  * nothing - the odd version checking needs a version change to stick to
+    Archetypes version again.
+    [yenzenz]
+
+1.3.5-final02 - 2005-08-01
+==========================
+
+ * nothing again, need to stick to Archetypes version
+   [yenzenz]
+
+
+1.3.5-final - 2005-07-17
+========================
+
+  * Added Five/Zope3 interface bridges and implements
+    [tiran]
+
+1.3.4-final - 2005-07-06
+========================
+
+  * added icons for openoffice.org files
+    [yenzenz]
+
+1.3.3-final06 - 2005-05-20
+==========================
+
+  * nothing (I hate to write this. But the odd version checking needs it).
+    [yenzenz]
+
+1.3.3-final-02 - 2005-03-25
+===========================
+
+  * nothing
+
+1.3.3-final - 2005-03-05
+========================
+
+ * More a workaround than a fix for [ 1056252 ] Content type algorithm
+   can be confused.
+   [tiran]
+
+ * workaround for [ 1068001 ] BaseUnit Encoding Error: macintosh
+   [yenzenz]
+
+ * In the case all else fails, try to resort to guess_content_type so
+   that at least we don't get 'text/plain' when the file is in fact a
+   binary file.
+   [dreamcatcher]
+
+1.3.2-5 - 2004-09-30
+====================
+
+ * nothing
+
+1.3.2-4 - 2004-09-30
+====================
+
+ * nothing
+
+1.3.2-3 - 2004-09-25
+====================
+
+ * nothing
+
+1.3.2-2 - 2004-09-17
+====================
+
+ * nothing
+
+1.3.2-1 - 2004-09-04
+====================
+
+ * Cleaned up major parts of PT by removing the python only implementation which
+   was broken anyway
+   [tiran]
+
+1.3.1-1 - 2004-08-16
+====================
+
+ * Added text/x-html-safe mime type for new transformation
+   [tiran]
+
+ * Don't return acquisition wrapped mimetype items beause they may lead to
+   memory leaks.
+   [tiran]
+
+1.3.0-3 - 2004-08-06
+====================
+
+ * Added text/wiki mime type
+   [tiran]
+
+ * Don't log redefine warning if the currrent and the new object are equal
+   [tiran]
+
+ * initialize() MTR on __setstate__ aka when the MTR is loaded from ZODB.
+   [tiran]
+
+1.3.0-2 - 2004-07-29
+====================
+
+ * Nothing changed

product/MimetypesRegistry/LICENSE.txt

+Copyright (c) 2002-2003, Benjamin Saller <bcsaller@ideasuite.com>, and 
+	                the respective authors.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.	
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following disclaimer
+      in the documentation and/or other materials provided with the
+      distribution. 
+    * Neither the name of Archetypes nor the names of its contributors
+      may be used to endorse or promote products derived from this software
+      without specific prior written permission. 
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+

product/MimetypesRegistry/MimeTypeItem.py

+import os
+
+from Acquisition import Explicit
+from OFS.SimpleItem import Item
+from AccessControl import ClassSecurityInfo
+from Globals import Persistent, InitializeClass
+
+from Products.CMFCore.permissions import ManagePortal
+from Products.MimetypesRegistry.interfaces import IMimetype
+from Products.MimetypesRegistry.common import MimeTypeException
+
+
+class MimeTypeItem(Persistent, Explicit, Item):
+    security = ClassSecurityInfo()
+    __implements__ = (IMimetype,)
+
+    extensions = ()
+    globs = ()
+
+    def __init__(self, name='', mimetypes=None, extensions=None,
+                 binary=None, icon_path='', globs=None):
+        if name:
+            self.__name__ = self.id = name
+        if mimetypes is not None:
+            self.mimetypes = mimetypes
+        if extensions is not None:
+            self.extensions = extensions
+        if binary is not None:
+            self.binary = binary
+        if globs is not None:
+            self.globs = globs
+        self.icon_path = icon_path or guess_icon_path(self)
+
+    def __str__(self):
+        return self.normalized()
+
+    def __repr__(self):
+        return "<mimetype %s>" % self.mimetypes[0]
+
+    def __cmp__(self, other):
+        try:
+            if isinstance(other, mimetype):
+                other = other.normalized()
+        except:
+            pass
+        return not (other in self.mimetypes)
+
+    def __hash__(self):
+        return hash(self.name())
+
+    security.declarePublic('name')
+    def name(self):
+        """ The name of this object """
+        return self.__name__
+
+    security.declarePublic('major')
+    def major(self):
+        """ return the major part of the RFC-2046 name for this mime type """
+        return self.normalized().split('/', 1)[0]
+
+    security.declarePublic('minor')
+    def minor(self):
+        """ return the minor part of the RFC-2046 name for this mime type """
+        return self.normalized().split('/', 1)[1]
+
+    security.declarePublic('normalized')
+    def normalized(self):
+        """ return the main RFC-2046 name for this mime type
+
+        e.g. if this object has names ('text/restructured', 'text-x-rst')
+        then self.normalized() will always return the first form.
+        """
+        return self.mimetypes[0]
+
+    security.declareProtected(ManagePortal, 'edit')
+    def edit(self, name, mimetypes, extensions, icon_path,
+             binary=0, globs=None, REQUEST=None):
+        """edit this mime type"""
+        # if mimetypes and extensions are string instead of lists,
+        # split them on new lines
+        if isinstance(mimetypes, basestring):
+            mimetypes = [mts.strip() for mts in mimetypes.split('\n')
+                         if mts.strip()]
+        if isinstance(extensions, basestring):
+            extensions = [mts.strip() for mts in extensions.split('\n')
+                          if mts.strip()]
+        if isinstance(globs, basestring):
+            globs = [glob.strip() for glob in globs.split('\n')
+                     if glob.strip()]
+        self.__name__ = self.id = name
+        self.mimetypes = mimetypes
+        self.globs = globs
+        self.extensions = extensions
+        self.binary = binary
+        self.icon_path = icon_path
+        if REQUEST is not None:
+            REQUEST['RESPONSE'].redirect(self.absolute_url()+'/manage_main')
+
+InitializeClass(MimeTypeItem)
+
+
+ICONS_DIR = os.path.join(os.path.dirname(__file__), 'skins', 'mimetypes_icons')
+
+def guess_icon_path(mimetype, icons_dir=ICONS_DIR, icon_ext='png'):
+    if mimetype.extensions:
+        for ext in mimetype.extensions:
+            icon_path = '%s.%s' % (ext, icon_ext)
+            if os.path.exists(os.path.join(icons_dir, icon_path)):
+                return icon_path
+    icon_path = '%s.png' % mimetype.major()
+    if os.path.exists(os.path.join(icons_dir, icon_path)):
+        return icon_path
+    return 'unknown.png'

product/MimetypesRegistry/MimeTypesTool.py

+# backward compatibility
+from Products.MimetypesRegistry.MimeTypesRegistry import MimeTypesRegistry as MimeTypesTool

product/MimetypesRegistry/README

+Mimetypes Registry
+=================
+
+ * mimetypes_registry (the mimetypes tool) : handle mime types information
+
+ * portal_transform (the transform tool) : handle transformation of data from a
+ mime type to another
+
+Documentation
+-------------
+See the *docs* directory in this package.
+
+
+Mailing-list
+------------
+Discussion about this products occurs to the archetypes mailing list :
+http://sourceforge.net/mail/?group_id=75272
+or on the #plone channel of irc.freenode.net.
+
+
+Authors
+-------
+
+Benjamin Saller <bcsaller@yahoo.com>
+Sidnei da Silva  <sidnei@x3ng.com>
+Sylvain Thénault <sylvain.thenault@logilab.fr>
+Christian Heimes <tiran@cheimes.de>

product/MimetypesRegistry/__init__.py

+import os.path
+__version__ = open(os.path.join(__path__[0], 'version.txt')).read().strip()
+
+from Products.MimetypesRegistry import MimeTypesRegistry
+from Products.MimetypesRegistry.common import skins_dir
+
+GLOBALS = globals()
+PKG_NAME = 'MimetypesRegistry'
+
+tools = (
+    MimeTypesRegistry.MimeTypesRegistry,
+    )
+
+from Products.MimetypesRegistry import mime_types
+
+# TODO: figure out if this is used/needed anywhere
+import sys
+from Products.MimetypesRegistry import MimeTypeItem
+sys.modules['Products.MimetypesRegistry.zope.MimeTypeItem'] = MimeTypeItem
+# end TODO
+
+def initialize(context):
+    from Products.CMFCore.DirectoryView import registerDirectory
+    registerDirectory(skins_dir, GLOBALS)
+
+    from Products.CMFCore import utils
+    utils.ToolInit("%s Tool" % PKG_NAME, 
+                   tools=tools,
+                   icon="tool.gif",
+                   ).initialize(context)

product/MimetypesRegistry/__pkginfo__.py

+from Products import MimetypesRegistry as PRODUCT
+import os.path
+
+version=PRODUCT.__version__
+modname=PRODUCT.__name__
+
+# (major, minor, patchlevel, release info) where release info is:
+# -99 for alpha, -49 for beta, -19 for rc and 0 for final
+# increment the release info number by one e.g. -98 for alpha2
+
+major, minor, bugfix =  version.split('.')[:3]
+bugfix, release = bugfix.split('-')[:2]
+
+relinfo=-99 #alpha
+if 'beta' in release:
+    relinfo=-49
+if 'rc' in release:
+    relinfo=-19
+if 'final' in release:
+    relinfo=0
+
+numversion = (int(major), int(minor), int(bugfix), relinfo)
+
+license = 'BSD like'
+license_text = open(os.path.join(PRODUCT.__path__[0], 'LICENSE.txt')).read()
+copyright = '''Copyright (c) 2003 LOGILAB S.A. (Paris, FRANCE)'''
+
+author = "Archetypes developement team"
+author_email = "archetypes-devel@lists.sourceforge.net"
+
+short_desc = "MIME types registry for the CMF"
+long_desc = """This package provides a new CMF tools in order to
+make MIME types guessings. You will find more info in the package's
+README and docs directory.
+.
+It's part of the Archetypes project, but the only requirement to use it
+is to have a CMF based site. If you are using Archetypes, this package
+replaces the transform package.
+.
+Notice this package can also be used as a standalone Python package. If
+you've downloaded the Python distribution, you can't make it a Zope
+product since Zope files have been removed from this distribution.
+"""
+
+web = "http://plone.org/products/archetypes"
+ftp = ""
+mailing_list = "archetypes-devel@lists.sourceforge.net"
+
+debian_name = "zope-cmfmtr"
+debian_maintainer = "Christian Heimes (?)"
+debian_maintainer_email = "tiran@cheimes.de"
+debian_handler = "zope"

product/MimetypesRegistry/base_zope.py

+"""some common utilities
+"""
+
+FB_REGISTRY = None
+
+# base class
+from ExtensionClass import Base
+from Acquisition import aq_base
+
+# logging function
+from zLOG import LOG, INFO
+def log(msg, severity=INFO, id='PortalTransforms'):
+    LOG(id, severity, msg)
+
+# directory where template for the ZMI are located
+import os.path
+_www = os.path.join(os.path.dirname(__file__), 'www')
+skins_dir = os.path.join(os.path.dirname(__file__), 'skins')
+
+# list and dict classes to use
+from Globals import PersistentMapping as DictClass
+try:
+    from ZODB.PersistentList import PersistentList as ListClass
+except ImportError:
+    from persistent.list import PersistentList as ListClass
+
+# interfaces
+try:
+    # Zope >= 2.6
+    from Interface import Interface, Attribute
+except ImportError:
+    # Zope < 2.6
+    from Interface import Base as Interface, Attribute
+
+def implements(object, interface):
+    return interface.isImplementedBy(object)
+
+# getToolByName
+from Products.CMFCore.utils import getToolByName as _getToolByName
+_marker = []
+
+def getToolByName(context, name, default=_marker):
+    global FB_REGISTRY
+    tool = _getToolByName(context, name, default)
+    if name == 'mimetypes_registry' and tool is default:
+        if FB_REGISTRY is None:
+            from Products.MimetypesRegistry.MimeTypesRegistry \
+                 import MimeTypesRegistry
+            FB_REGISTRY = MimeTypesRegistry()
+        tool = FB_REGISTRY
+    return tool
+
+from zExceptions import BadRequest
+
+__all__ = ('Base', 'log', 'DictClass', 'ListClass', 'getToolByName', 'aq_base',
+           'Interface', 'Attribute', 'implements', 'skins_dir', '_www',
+           'BadRequest', )

product/MimetypesRegistry/bridge.zcml

+<configure
+    xmlns="http://namespaces.zope.org/five"
+    >
+ 
+  <bridge
+      zope2=".interfaces.IMimetype"
+      package=".z3.interfaces"
+      name="IMimetype"
+      /> 
+
+  <bridge
+      zope2=".interfaces.IClassifier"
+      package=".z3.interfaces"
+      name="IClassifier"
+      /> 
+
+  <bridge
+      zope2=".interfaces.ISourceAdapter"
+      package=".z3.interfaces"
+      name="ISourceAdapter"
+      /> 
+
+  <bridge
+      zope2=".interfaces.IMimetypesRegistry"
+      package=".z3.interfaces"
+      name="IMimetypesRegistry"
+      /> 
+
+</configure>

product/MimetypesRegistry/common.py

+"""some common utilities
+"""
+from time import time
+from types import UnicodeType, StringType
+
+STRING_TYPES = (UnicodeType, StringType)
+
+class MimeTypeException(Exception):
+    pass
+
+# logging function
+from zLOG import LOG, INFO
+def log(msg, severity=INFO, id='MimetypesRegistry'):
+    LOG(id, severity, msg)
+
+# directory where template for the ZMI are located
+import os.path
+_www = os.path.join(os.path.dirname(__file__), 'www')
+skins_dir = os.path.join(os.path.dirname(__file__), 'skins')

product/MimetypesRegistry/configure.zcml

+<configure
+    xmlns="http://namespaces.zope.org/zope"
+    >
+
+  <include file="bridge.zcml"/>
+
+  <include file="implements.zcml"/>
+
+</configure>
+

product/MimetypesRegistry/encoding.py

+import re
+import encodings
+from Products.MimetypesRegistry.common import log
+
+EMACS_ENCODING_RGX = re.compile('[^#]*[#\s]*-\*-\s*coding: ([^\s]*)\s*-\*-\s*')
+VIM_ENCODING_RGX = re.compile('[^#]*[#\s]*vim:fileencoding=\s*([^\s]*)\s*')
+XML_ENCODING_RGX = re.compile('<\?xml version=[^\s]*\s*encoding=([^\s]*)\s*\?>')
+CHARSET_RGX = re.compile('charset=([^\s"]*)')
+
+def guess_encoding(buffer):
+    """Better guess encoding method
+    
+    It checks if python supports the encoding
+    """
+    encoding = _guess_encoding(buffer)
+    # step 1: if the encoding was detected, use the lower() because python
+    # is using lower case names for encodings
+    if encoding and isinstance(encoding, basestring):
+        #encoding = encoding.lower()
+        pass
+    else:
+        return None
+    # try to find an encoding function for the encoding
+    # if None is returned or an exception is raised the encoding is invalid
+    try:
+        result = encodings.search_function(encoding.lower())
+    except:
+        # XXX log
+        result = None
+    
+    if result:
+        # got a valid encoding
+        return encoding
+    else:
+        return None
+    
+
+def _guess_encoding(buffer):
+    """try to guess encoding from a buffer
+
+    FIXME: it could be mime type driven but it seems less painful like that
+    """
+    assert type(buffer) is type(''), type(buffer)
+    # default to ascii on empty buffer
+    if not buffer:
+        return 'ascii'
+
+    # check for UTF-8 byte-order mark
+    if buffer.startswith('\xef\xbb\xbf'):
+        return 'UTF-8'
+
+    first_lines = buffer.split('\n')[:2]
+    for line in first_lines:
+        # check for emacs encoding declaration
+        m = EMACS_ENCODING_RGX.match(line)
+        if m is not None:
+            return m.group(1)
+        # check for vim encoding declaration
+        m = VIM_ENCODING_RGX.match(line)
+        if m is not None:
+            return m.group(1)
+
+    # check for xml encoding declaration
+    if first_lines[0].startswith('<?xml'):
+        m = XML_ENCODING_RGX.match(first_lines[0])
+        if m is not None:
+            return m.group(1)[1:-1]
+        # xml files with no encoding declaration default to UTF-8
+        return 'UTF-8'
+
+    # try to get charset declaration
+    # FIXME: should we check it's html before ?
+    m = CHARSET_RGX.search(buffer)
+    if m is not None:
+        return m.group(1)
+    return None

product/MimetypesRegistry/implements.zcml

+<configure
+    xmlns="http://namespaces.zope.org/five"
+    >
+
+  <!-- TODO: more -->
+
+</configure>

product/MimetypesRegistry/interfaces.py

+from Interface import Interface, Attribute
+
+class IMimetype(Interface):
+    """Specification for dealing with mimetypes RFC-2046 style"""
+
+#     mimetypes = Attribute("List of mimetypes in the RFC-2046 format")
+#     extensions = Attribute("""List of extensions mapped to this
+#     mimetype w/o the leading .""")
+
+#     binary = Attribute("""Boolean indicating if the mimetype should be
+#     treated as binary (and not human readable)""")
+
+    def name(self):
+        """return the Human readable name of the mimetype"""
+
+    def major(self):
+        """ return the major part of the RFC-2046 name for this mime type """
+
+    def minor(self):
+        """ return the minor part of the RFC-2046 name for this mime type """
+
+    def normalized(self):
+        """ return the main RFC-2046 name for this mime type
+
+        e.g. if this object has names ('text/restructured', 'text-x-rst')
+        then self.normalized() will always return the first form.
+        """
+
+class IClassifier(Interface):
+    """Optional mixin interface for imimetype, code to test if the
+    mimetype is present in data
+    """
+    def classify(data):
+        """ boolean indicating if the data fits the mimetype"""
+
+
+class ISourceAdapter(Interface):
+
+    def __call__(data, **kwargs):
+        """convert data to unicode, may take optional kwargs to aid in
+        conversion"""
+
+
+class IMimetypesRegistry(Interface):
+
+    def classify(data, mimetype=None, filename=None):
+        """return a content type for this data or None
+        None should rarely be returned as application/octet can be
+        used to represent most types
+        """
+
+    def lookup(mimetypestring):
+        """Lookup for imimetypes object matching mimetypestring
+
+        mimetypestring may have an empty minor part or containing a wildcard (*)
+        mimetypestring may be an imimetype object (in this case it will be
+        returned unchanged, else it should be a RFC-2046 name
+
+        return a list of mimetypes objects associated with the RFC-2046 name
+        return an empty list if no one is known.
+        """
+
+    def lookupExtension(filename):
+        """ return the mimetypes object associated with the file's extension
+        return None if it is not known.
+
+        filename maybe a file name like 'content.txt' or an extension like 'rest'
+        """
+
+    def mimetypes():
+        """return all defined mime types, each one implements at least imimetype
+        """
+
+    def list_mimetypes():
+        """return all defined mime types, as string"""

product/MimetypesRegistry/mime_types/__init__.py

+import mtr_mimetypes
+import py_mimetypes
+import smi_mimetypes
+import suppl_mimetypes
+import magic
+
+from mtr_mimetypes import *
+
+def initialize(registry):
+    mtr_mimetypes.initialize(registry)
+    smi_mimetypes.initialize(registry)
+    suppl_mimetypes.initialize(registry)
+    py_mimetypes.initialize(registry)

product/MimetypesRegistry/mime_types/mtr_mimetypes.py

+from Products.MimetypesRegistry.interfaces import IClassifier
+from Products.MimetypesRegistry.MimeTypeItem import MimeTypeItem
+from Products.MimetypesRegistry.common import MimeTypeException
+
+from types import InstanceType
+import re
+
+class text_plain(MimeTypeItem):
+
+    __implements__ = MimeTypeItem.__implements__
+    __name__   = "Plain Text"
+    mimetypes  = ('text/plain',)
+    extensions = ('txt',)
+    binary     = 0
+
+class text_pre_plain(MimeTypeItem):
+
+    __implements__ = MimeTypeItem.__implements__
+    __name__   = "Pre-formatted Text (<pre>)"
+    mimetypes  = ('text/plain-pre',)
+    extensions = ()
+    binary     = 0
+
+class text_structured(MimeTypeItem):
+
+    __implements__ = MimeTypeItem.__implements__
+    __name__   = "Structured Text"
+    mimetypes  = ('text/structured',)
+    extensions = ('stx',)
+    binary     = 0
+
+class text_rest(MimeTypeItem):
+
+    __implements__ = MimeTypeItem.__implements__
+    __name__   = "reStructured Text"
+    mimetypes  = ("text/x-rst", "text/restructured",)
+    extensions = ("rst", "rest", "restx") #txt?
+    binary     = 0
+
+class text_python(MimeTypeItem):
+
+    __implements__ = MimeTypeItem.__implements__
+    __name__   = "Python Source"
+    mimetypes  = ("text/python-source", "text/x-python",)
+    extensions = ("py",)
+    binary     = 0
+
+class text_wiki(MimeTypeItem):
+
+    __implements__ = MimeTypeItem.__implements__
+    __name__   = "Wiki text"
+    mimetypes  = ("text/wiki",)
+    extensions = ()
+    binary     = 0
+
+class application_rtf(MimeTypeItem):
+
+    __implements__ = MimeTypeItem.__implements__
+    __name__   = 'Rich Text Format (RTF)'
+    mimetypes  = ('application/rtf',)
+    extensions = ('rtf',)
+    binary     = 1
+
+class application_msword(MimeTypeItem):
+
+    __implements__ = MimeTypeItem.__implements__
+    __name__   = "Microsoft Word Document"
+    mimetypes  = ('application/msword',)
+    extensions = ('doc',)
+    binary     = 1
+
+class text_xml(MimeTypeItem):
+
+    __implements__ = MimeTypeItem.__implements__ + (IClassifier,)
+
+    __name__   = "Extensible Markup Language (XML)"
+    mimetypes  = ('text/xml',)
+    extensions = ('xml',)
+    binary     = 0
+
+    def classify(self, data):
+        m = re.search('^\s*<\\?xml.*\\?>', data)
+        if m:
+            return 1 # True
+        return None  # False
+
+class application_octet_stream(MimeTypeItem):
+    """we need to be sure this one exists"""
+    __name__   = "Octet Stream"
+    mimetypes = ('application/octet-stream',)
+    binary     = 1
+    extensions = ()
+
+class text_html(MimeTypeItem):
+
+    __implements__ = MimeTypeItem.__implements__
+    __name__   = "HTML"
+    mimetypes  = ('text/html',)
+    extensions = ('html', 'htm')
+    binary     = 0
+
+class text_html_safe(MimeTypeItem):
+
+    __implements__ = MimeTypeItem.__implements__
+    __name__   = "Safe HTML"
+    mimetypes  = ('text/x-html-safe',)
+    extensions = ()
+    binary     = 0
+
+reg_types = [
+    text_plain,
+    text_pre_plain,
+    application_msword,
+    text_xml,
+    text_structured,
+    text_rest,
+    text_python,
+    text_wiki,
+    application_octet_stream,
+    application_rtf,
+    text_html,
+    text_html_safe,
+    ]
+
+def initialize(registry):
+    for mt in reg_types:
+        if type(mt) != InstanceType:
+            mt = mt()
+        registry.register(mt)
+
+__all__ = tuple([cls.__name__ for cls in reg_types])

product/MimetypesRegistry/mime_types/py_mimetypes.py

+import os.path
+from Products.MimetypesRegistry.MimeTypeItem import MimeTypeItem
+from Products.MimetypesRegistry.MimeTypeItem import guess_icon_path
+from Products.MimetypesRegistry.common import MimeTypeException
+
+try:
+    from zope.contenttype import add_files
+except ImportError: # BBB: Zope < 2.10
+    try:
+        from zope.app.content_types import add_files
+    except ImportError: # BBB: Zope < 2.9
+        from OFS.content_types import add_files
+
+import mimetypes as pymimetypes
+
+mimes_initialized = False
+
+def mimes_initialize():
+    global mimes_initialized
+    if mimes_initialized:
+        return
+    mimes_initialized = True
+    # Augment known mime-types.
+    here = os.path.dirname(os.path.abspath(__file__))
+    add_files([os.path.join(here, 'mime.types')])
+
+# don't register the mimetype from python mimetypes if matching on of
+# this extensions.
+skip_extensions = (
+    )
+
+def initialize(registry):
+    # Find things that are not in the specially registered mimetypes
+    # and add them using some default policy, none of these will impl
+    # iclassifier
+
+    # Read our included mime.types file, in addition to whatever the
+    # mimetypes python module might have found.
+    mimes_initialize()
+
+    # Initialize from registry known mimetypes if we are on Windows
+    # and pywin32 is available.
+    try:
+        from windows_mimetypes import initialize
+        initialize()
+    except ImportError:
+        pass
+    
+    for ext, mt in pymimetypes.types_map.items():
+        if ext[0] == '.':
+            ext = ext[1:]
+
+        if registry.lookupExtension(ext):
+            continue
+        if ext in skip_extensions:
+            continue
+
+        try:
+            mto =  registry.lookup(mt)
+        except MimeTypeException:
+            # malformed MIME type
+            continue
+        if mto:
+            mto = mto[0]
+            if not ext in mto.extensions:
+                registry.register_extension(ext, mto)
+                mto.extensions += (ext, )
+                # here we guess icon path again, to find icon match the new ext
+                mto.icon_path = guess_icon_path(mto)
+            continue
+        isBin = mt.split('/', 1)[0] != "text"
+        registry.register(MimeTypeItem(mt, (mt,), (ext,), isBin))

product/MimetypesRegistry/mime_types/smi_mimetypes.py

+import os
+from xml.sax import parse
+from xml.sax.handler import ContentHandler
+
+DIR = os.path.dirname(__file__)
+SMI_NAME = "freedesktop.org.xml"
+SMI_FILE = os.path.join(DIR, SMI_NAME)
+
+class SharedMimeInfoHandler(ContentHandler):
+
+    current = None
+    collect_comment = None
+
+    def __init__(self):
+        ContentHandler.__init__(self)
+        self.mimes = []
+
+    def startElement(self, name, attrs):
+        if name in ('mime-type',):
+            current = {'type': attrs['type'],
+                       'comments': {},
+                       'globs': [],
+                       'aliases': []}
+            self.mimes.append(current)
+            self.current = current
+            return
+        if name in ('comment',):
+            # If no lang, assume 'en'
+            lang = attrs.get('xml:lang', 'en')
+            if lang not in ('en',):
+                # Ignore for now.
+                return
+            self.__comment_buffer = []
+            self.__comment_lang = lang
+            self.collect_comment = True
+            return
+        if name in ('glob',):
+            globs = self.current['globs']
+            globs.append(attrs['pattern'])
+            return
+        if name in ('alias',):
+            aliases = self.current['aliases']
+            aliases.append(attrs['type'])
+
+    def endElement(self, name):
+        if self.collect_comment and name in ('comment',):
+            self.collect_comment = False
+            lang = self.__comment_lang
+            comment = u''.join(self.__comment_buffer)
+            if not comment:
+                comment = self.current['type']
+            self.current['comments'][lang] = comment
+
+    def characters(self, contents):
+        if self.collect_comment:
+            self.__comment_buffer.append(contents)
+
+def readSMIFile(infofile):
+    """Reads a shared mime info XML file
+    """
+    handler = SharedMimeInfoHandler()
+    parse(infofile, handler)
+    return handler.mimes
+
+mimetypes = readSMIFile(SMI_FILE)
+
+def initialize(registry):
+    global mimetypes
+    from Products.MimetypesRegistry.MimeTypeItem import MimeTypeItem
+    from Products.MimetypesRegistry.common import MimeTypeException
+    # Find things that are not in the specially registered mimetypes
+    # and add them using some default policy, none of these will impl
+    # iclassifier
+    for res in mimetypes:
+        mt = str(res['type'])
+        mts = (mt,) + tuple(res['aliases'])
+
+        # check the mime type
+        try:
+            mto =  registry.lookup(mt)
+        except MimeTypeException:
+            # malformed MIME type
+            continue
+
+        name = str(res['comments'].get(u'en', mt))
+
+        # build a list of globs
+        globs = []
+        for glob in res['globs']:
+            if registry.lookupGlob(glob):
+                continue
+            else:
+                globs.append(glob)
+
+        if mto:
+            mto = mto[0]
+            for glob in globs:
+                if not glob in mto.globs:
+                    mto.globs = list(mto.globs) + [glob]
+                    registry.register_glob(glob, mto)
+            for mt in mts:
+                if not mt in mto.mimetypes:
+                    mto.mimetypes = list(mto.mimetypes) + [mt]
+                    registry.register_mimetype(mt, mto)
+        else:
+            isBin = mt.split('/', 1)[0] != "text"
+            mti = MimeTypeItem(name, mimetypes=mts,
+                               binary=isBin,
+                               globs=globs)
+            registry.register(mti)

product/MimetypesRegistry/mime_types/suppl_mimetypes.py

+from Products.MimetypesRegistry.MimeTypeItem import MimeTypeItem
+from Products.MimetypesRegistry.common import MimeTypeException
+
+
+map = {
+    # '.extension' : 'mimetype',
+    '.svg' : 'image/svg+xml', # scaleable vector graphics
+    '.pjpg' : 'image/pjpeg', # scaleable vector graphics
+    
+}
+
+def initialize(registry):
+    #Find things that are not in the specially registered mimetypes
+    #and add them using some default policy, none of these will impl
+    #iclassifier
+    for ext, mt in map.items():
+        if ext[0] == '.':
+            ext = ext[1:]
+        
+        if registry.lookupExtension(ext):
+            continue
+
+        try:
+            mto =  registry.lookup(mt)
+        except MimeTypeException:
+            # malformed MIME type
+            continue
+        if mto:
+            mto = mto[0]
+            if not ext in mto.extensions:
+                registry.register_extension(ext, mto)
+                mto.extensions += (ext, )
+            continue
+        isBin = mt.split('/', 1)[0] != "text"
+        registry.register(MimeTypeItem(mt, (mt,), (ext,), isBin))

product/MimetypesRegistry/mime_types/windows_mimetypes.py

+# Utilities for mime-types and the Windows registry.
+import _winreg
+import win32api
+import win32con
+import mimetypes
+
+import logging
+logger = logging.getLogger('mimetypes.win32')
+
+# "safely" query a value, returning a default when it doesn't exist.
+def _RegQueryValue(key, value, default=None):
+    try:
+        data, typ = win32api.RegQueryValueEx(key, value)
+    except win32api.error:
+        return default
+    if typ == win32con.REG_EXPAND_SZ:
+        data = win32api.ExpandEnvironmentStrings(data)
+    if type in (win32con.REG_EXPAND_SZ, win32con.REG_SZ):
+        # Occasionally see trailing \0 chars.
+        data = data.rstrip('\0')
+    return data
+
+def get_desc_for_mimetype(mime_type):
+    try:
+        hk = win32api.RegOpenKey(win32con.HKEY_CLASSES_ROOT,
+                                 r"MIME\Database\Content Type\\" + mime_type)
+        desc = _RegQueryValue(hk, "")
+    except win32api.error, details:
+        logger.info("win32api error fetching description for mime-type %r: %s",
+                     mime_type, details)
+        desc = None
+    logger.debug("mime-type %s has description %s", mime_type, desc)
+    return desc
+
+def get_ext_for_mimetype(mime_type):
+    try:
+        hk = win32api.RegOpenKey(win32con.HKEY_CLASSES_ROOT,
+                                 r"MIME\Database\Content Type\\" + mime_type)
+        ext = _RegQueryValue(hk, "Extension")
+    except win32api.error, details:
+        logger.info("win32api error fetching extension for mime-type %r: %s",
+                     mime_type, details)
+        ext = None
+    logger.debug("mime-type %s has extension %s", mime_type, ext)
+    return ext
+
+def get_mime_types():
+    try:
+        hk = win32api.RegOpenKey(win32con.HKEY_CLASSES_ROOT,
+                                 r"MIME\Database\Content Type")
+        items = win32api.RegEnumKeyEx(hk)
+    except win32api.error, details:
+        logger.info("win32api error fetching mimetypes: %s",
+                    details)
+        items = []
+    return [i[0] for i in items if i[0]]
+
+def normalize(mt):
+    # Some mimetypes might have extra ';q=value' params.
+    return mt.lower().split(';')[0]
+
+def initialize():
+    if not mimetypes.inited:
+        mimetypes.init()
+
+    for mt in get_mime_types():
+        ext = get_ext_for_mimetype(mt)
+        if ext is None:
+            continue
+        if not mimetypes.types_map.has_key(ext):
+            mimetypes.add_type(normalize(mt), ext)
+
+if __name__=='__main__':
+    for mt in get_mime_types():
+        ext = get_ext_for_mimetype(mt)
+        desc = get_desc_for_mimetype(mt)
+        print "%s (%s) - %s" % (mt.lower(), desc, ext)
+    import code; code.interact(local=locals())

product/MimetypesRegistry/tests/__init__.py

+"""import this"""

product/MimetypesRegistry/tests/framework.py

+##############################################################################
+#
+# ZopeTestCase 
+#
+# COPY THIS FILE TO YOUR 'tests' DIRECTORY.
+#
+# This version of framework.py will use the SOFTWARE_HOME
+# environment variable to locate Zope and the Testing package.
+#
+# If the tests are run in an INSTANCE_HOME installation of Zope,
+# Products.__path__ and sys.path with be adjusted to include the
+# instance's Products and lib/python directories respectively.
+#
+# If you explicitly set INSTANCE_HOME prior to running the tests,
+# auto-detection is disabled and the specified path will be used 
+# instead.
+#
+# If the 'tests' directory contains a custom_zodb.py file, INSTANCE_HOME
+# will be adjusted to use it.
+#
+# If you set the ZEO_INSTANCE_HOME environment variable a ZEO setup 
+# is assumed, and you can attach to a running ZEO server (via the 
+# instance's custom_zodb.py).
+#
+##############################################################################
+#
+# The following code should be at the top of every test module:
+#
+# import os, sys
+# if __name__ == '__main__':
+#     execfile(os.path.join(sys.path[0], 'framework.py'))
+#
+# ...and the following at the bottom:
+#
+# if __name__ == '__main__':
+#     framework()
+#
+##############################################################################
+
+__version__ = '0.2.3'
+
+# Save start state
+#
+__SOFTWARE_HOME = os.environ.get('SOFTWARE_HOME', '')
+__INSTANCE_HOME = os.environ.get('INSTANCE_HOME', '')
+
+if __SOFTWARE_HOME.endswith(os.sep):
+    __SOFTWARE_HOME = os.path.dirname(__SOFTWARE_HOME)
+
+if __INSTANCE_HOME.endswith(os.sep):
+    __INSTANCE_HOME = os.path.dirname(__INSTANCE_HOME)
+
+# Find and import the Testing package
+#
+if not sys.modules.has_key('Testing'):
+    p0 = sys.path[0]
+    if p0 and __name__ == '__main__':
+        os.chdir(p0)
+        p0 = ''
+    s = __SOFTWARE_HOME
+    p = d = s and s or os.getcwd()
+    while d:
+        if os.path.isdir(os.path.join(p, 'Testing')):
+            zope_home = os.path.dirname(os.path.dirname(p))
+            sys.path[:1] = [p0, p, zope_home]
+            break
+        p, d = s and ('','') or os.path.split(p)
+    else:
+        print 'Unable to locate Testing package.',
+        print 'You might need to set SOFTWARE_HOME.'
+        sys.exit(1)
+
+import Testing, unittest
+execfile(os.path.join(os.path.dirname(Testing.__file__), 'common.py'))
+
+# Include ZopeTestCase support
+#
+if 1:   # Create a new scope
+
+    p = os.path.join(os.path.dirname(Testing.__file__), 'ZopeTestCase')
+
+    if not os.path.isdir(p):
+        print 'Unable to locate ZopeTestCase package.',
+        print 'You might need to install ZopeTestCase.'
+        sys.exit(1)
+
+    ztc_common = 'ztc_common.py'
+    ztc_common_global = os.path.join(p, ztc_common)
+
+    f = 0
+    if os.path.exists(ztc_common_global):
+        execfile(ztc_common_global)
+        f = 1
+    if os.path.exists(ztc_common):
+        execfile(ztc_common)
+        f = 1
+
+    if not f:
+        print 'Unable to locate %s.' % ztc_common
+        sys.exit(1)
+
+# Debug
+#
+print 'SOFTWARE_HOME: %s' % os.environ.get('SOFTWARE_HOME', 'Not set')
+print 'INSTANCE_HOME: %s' % os.environ.get('INSTANCE_HOME', 'Not set')
+sys.stdout.flush()
+

product/MimetypesRegistry/tests/input/rest1.rst

+
+This is a test of the *reST* transform
+    o one
+    o two
+    o three
+

product/MimetypesRegistry/tests/runalltests.py

+#
+# Runs all tests in the current directory
+#
+# Execute like:
+#   python runalltests.py
+#
+# Alternatively use the testrunner: 
+#   python /path/to/Zope/utilities/testrunner.py -qa
+#
+
+import os, sys
+if __name__ == '__main__':
+    execfile(os.path.join(sys.path[0], 'framework.py')) 
+
+import unittest
+TestRunner = unittest.TextTestRunner
+suite = unittest.TestSuite()
+
+tests = os.listdir(os.curdir)
+tests = [n[:-3] for n in tests if n.startswith('test') and n.endswith('.py')]
+
+for test in tests:
+    m = __import__(test)
+    if hasattr(m, 'test_suite'):
+        suite.addTest(m.test_suite())
+
+if __name__ == '__main__':
+    TestRunner().run(suite)
+

product/MimetypesRegistry/tests/runtests.sh.example

+#!/bin/bash
+#
+# example test runner shell script
+#
+
+# full path to the python interpretor
+export PYTHON="/usr/local/bin/python2.3"
+
+# path to ZOPE_HOME/lib/python
+export SOFTWARE_HOME="/opt/zope/releases/Zope-2_7-branch/lib/python"
+
+
+# path to your instance. Don't set it if you aren't having  instance
+export INSTANCE_HOME="/opt/zope/instances/plone21/"
+
+
+${PYTHON} runalltests.py

product/MimetypesRegistry/tests/test_encoding.py

+import os, sys
+if __name__ == '__main__':
+    execfile(os.path.join(sys.path[0], 'framework.py'))
+
+from Testing import ZopeTestCase
+from Products.Archetypes.tests.atsitetestcase import ATSiteTestCase
+
+from Products.MimetypesRegistry.encoding import guess_encoding
+
+class TestGuessEncoding(ATSiteTestCase):
+
+    def testUTF8(self):
+        e = guess_encoding('\xef\xbb\xbf any UTF-8 data')
+        self.failUnlessEqual(e, 'UTF-8')
+        e = guess_encoding(' any UTF-8 data \xef\xbb\xbf')
+        self.failUnlessEqual(e, None)
+
+    def testEmacs(self):
+        e = guess_encoding('# -*- coding: UTF-8  -*-')
+        self.failUnlessEqual(e, 'UTF-8')
+        e = guess_encoding('''
+        ### -*- coding: ISO-8859-1  -*-
+        ''')
+        self.failUnlessEqual(e, 'ISO-8859-1')
+        e = guess_encoding('''
+
+        ### -*- coding: ISO-8859-1  -*-
+        ''')
+        self.failUnlessEqual(e, None)
+
+    def testVim(self):
+        e = guess_encoding('# vim:fileencoding=UTF-8')
+        self.failUnlessEqual(e, 'UTF-8')
+        e = guess_encoding('''
+        ### vim:fileencoding=ISO-8859-1
+        ''')
+        self.failUnlessEqual(e, 'ISO-8859-1')
+        e = guess_encoding('''
+
+        ### vim:fileencoding= ISO-8859-1
+        ''')
+        self.failUnlessEqual(e, None)
+
+    def testXML(self):
+        e = guess_encoding('<?xml?>')
+        self.failUnlessEqual(e, 'UTF-8')
+        e = guess_encoding('''<?xml version="1.0" encoding="ISO-8859-1" ?>
+        ''')
+        self.failUnlessEqual(e, 'ISO-8859-1')
+        e = guess_encoding('''<?xml version="1.0" encoding="ISO-8859-1"?>
+        ''')
+        self.failUnlessEqual(e, 'ISO-8859-1')
+        e = guess_encoding('''<?xml version="1.0" encoding="ISO-8859-1"?><truc encoding="UTF-8">
+        </truc>
+        ''')
+        self.failUnlessEqual(e, 'ISO-8859-1')
+
+    def testHTML(self):
+        e = guess_encoding('''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
+<html>
+<head>
+<title>ASPN : Python Cookbook : Auto-detect XML encoding</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+    <meta name="robots" content="all" />
+    <meta name="description" content="ActiveState Open Source Programming tools for Perl Python XML xslt scripting with free trials. Quality development tools for programmers systems administrators database administrators network administrators and webmasters" />
+    <meta name="keywords" content="ActiveState,Perl,xml,xslt,mozilla,Open Source,Python,Perl for Win32,resources,PerlScript,ActivePerl,Programming,Programmers,Integrated,Development,Environment,SOAP,Linux,Solaris,Web,development,tools,free,software,download,support,Perl Resource Kit,System Administration,Sys Admin,WinNT,SQL,Oracle,Email,XML,Linux,Programming,perl,NT,2000,windows,Unix,Software,Security,   Administration,systems,windows,database,database,consulting,support,Microsoft,developer,resource,code,tutorials,IDE,Integrated development environment,developer,resources,tcl,php" />
+
+<link rel="stylesheet" href="/ASPN/aspn.css" />
+
+</head>
+
+<body bgcolor="#FFFFFF" leftmargin="0" topmargin="0" marginwidth="0" marginheight="0">
+charset=utf-8
+</body>
+</html> ''')
+        self.failUnlessEqual(e, 'iso-8859-1')
+
+
+    def test_broken_percent(self):
+        e = guess_encoding(
+r"""<pre>
+&lt;metal:block tal:define="dummy python:
+request.RESPONSE.setHeader('Content-Type',
+'text/html;;charset=%s' % charset)" /&gt;
+&lt;metal:block tal:define="dummy
+python:request.RESPONSE.setHeader('Content-Language', lang)"
+/
+&gt;
+</pre>
+"""
+    )
+        # unable to detect a valid encoding
+        self.failUnlessEqual(e, None) 
+
+
+def test_suite():
+    from unittest import TestSuite, makeSuite
+    suite = TestSuite()
+    suite.addTest(makeSuite(TestGuessEncoding))
+    return suite
+
+if __name__ == '__main__':
+    framework()

product/MimetypesRegistry/tests/test_magic.py

+import os, sys
+if __name__ == '__main__':
+    execfile(os.path.join(sys.path[0], 'framework.py'))
+
+from Testing import ZopeTestCase
+from Products.Archetypes.tests.atsitetestcase import ATSiteTestCase
+
+from Products.MimetypesRegistry.mime_types.magic import guessMime
+from utils import input_file_path
+
+samplefiles = [
+    ('OOoWriter', 'application/vnd.sun.xml.writer'),
+    ('OOoCalc', 'application/vnd.sun.xml.calc'),
+    ('sxw-ooo-trolltech', 'application/vnd.sun.xml.writer'), # file from limi
+    ('simplezip', 'application/zip'),
+]
+
+class TestGuessMagic(ATSiteTestCase):
+    
+    def afterSetUp(self):
+        ATSiteTestCase.afterSetUp(self)
+        self.registry = self.portal.mimetypes_registry
+
+    def test_guessMime(self):
+        for filename, expected in samplefiles:
+            file = open(input_file_path(filename))
+            data = file.read()
+            file.close()
+            
+            # use method direct
+            got = guessMime(data)
+            self.failUnlessEqual(got, expected)
+            
+            # use mtr-tool
+            got_from_tool = self.registry.classify(data)
+            self.failUnlessEqual(got_from_tool, expected)            
+            
+            # now cut it to the first 8k if greater
+            if len(data) > 8192:
+                data=data[:8192]
+                got_cutted = self.registry.classify(data)
+                self.failUnlessEqual(got_cutted, expected)          
+
+
+def test_suite():
+    from unittest import TestSuite, makeSuite
+    suite = TestSuite()
+    suite.addTest(makeSuite(TestGuessMagic))
+    return suite
+
+if __name__ == '__main__':
+    framework()

product/MimetypesRegistry/tests/test_mimetypes.py

+import os, sys
+if __name__ == '__main__':
+    execfile(os.path.join(sys.path[0], 'framework.py'))
+from Testing import ZopeTestCase
+from Products.Archetypes.tests.atsitetestcase import ATSiteTestCase
+
+from Products.MimetypesRegistry.mime_types import text_plain
+from Products.MimetypesRegistry.mime_types import text_xml
+from Products.MimetypesRegistry.mime_types import application_octet_stream
+from utils import input_file_path
+
+class TestMimeTypesclass(ATSiteTestCase):
+
+    def afterSetUp(self):
+        ATSiteTestCase.afterSetUp(self)
+        self.registry = self.portal.mimetypes_registry
+
+    def testClassify(self):
+        reg = self.registry
+        c = reg._classifiers()
+        self.failUnless(c[0].name().startswith("Extensible Markup Language"),
+                        c[0].name())
+
+        #Real XML
+        data = "<?xml version='1.0'?><foo>bar</foo>"
+        mt = reg.classify(data)
+        self.failUnless(isinstance(mt, text_xml), str(mt))
+
+        # with leading whitespace (http://dev.plone.org/archetypes/ticket/622)
+        # still valid xml
+        data = " <?xml version='1.0'?><foo>bar</foo>"
+        mt = reg.classify(data)
+        self.failUnless(isinstance(mt, text_xml), str(mt))
+        
+        # also #622: this is not xml
+        data = 'xml > plain text'
+        mt = reg.classify(data)
+        self.failUnless(str(mt) != 'text/xml')
+
+        #Passed in MT
+        mt = reg.classify(data, mimetype="text/plain")
+        self.failUnless(isinstance(mt, text_plain), str(mt))
+
+        #Passed in filename
+        mt = reg.classify(data, filename="test.xml")
+        self.failUnless(isinstance(mt, text_xml), str(mt))
+        mt = reg.classify(data, filename="test.jpg")
+        self.failUnlessEqual(str(mt), 'image/jpeg')
+
+        # use xml classifier
+        mt = reg.classify('<?xml ?>')
+        self.failUnless(isinstance(mt, text_xml), str(mt))
+
+        # test no data return default
+        mt = reg.classify('')
+        self.failUnless(isinstance(mt, text_plain), str(mt))
+        reg.defaultMimetype = 'text/xml'
+        mt = reg.classify('')
+        self.failUnless(isinstance(mt, text_xml), str(mt))
+
+        # test unclassifiable data and no stream flag (filename)
+        mt = reg.classify('xxx')
+        self.failUnless(isinstance(mt, text_plain), str(mt))
+
+        # test unclassifiable data and file flag
+        mt = reg.classify('baz', filename='xxx')
+        self.failUnless(isinstance(mt, application_octet_stream), str(mt))
+
+    def testExtension(self):
+        reg = self.registry
+        data = "<foo>bar</foo>"
+        mt = reg.lookupExtension(filename="test.xml")
+        self.failUnless(isinstance(mt, text_xml), str(mt))
+
+        mt = reg.classify(data, filename="test.foo")
+        self.failUnless(isinstance(mt, application_octet_stream), str(mt))
+
+        mt = reg.classify(data, filename="test.tgz")
+        self.failUnlessEqual(str(mt), 'application/x-tar')
+
+        mt = reg.classify(data, filename="test.tar.gz")
+        self.failUnlessEqual(str(mt), 'application/x-tar')
+
+        mt = reg.classify(data, filename="test.pdf.gz")
+        self.failUnlessEqual(str(mt), 'application/pdf')
+
+    def testFDOGlobs(self):
+        # The mime types here might only match if they match a glob on
+        # the freedesktop.org registry.
+        data = ''
+        reg = self.registry
+        mt = reg.classify(data, filename="test.anim1")
+        self.failUnlessEqual(str(mt), 'video/x-anim')
+
+        mt = reg.classify(data, filename="test.ini~")
+        self.failUnlessEqual(str(mt), 'application/x-trash')
+
+        mt = reg.classify(data, filename="test.ini%")
+        self.failUnlessEqual(str(mt), 'application/x-trash')
+
+        mt = reg.classify(data, filename="test.ini.bak")
+        self.failUnlessEqual(str(mt), 'application/x-trash')
+
+        mt = reg.classify(data, filename="test.f90")
+        self.failUnlessEqual(str(mt), 'text/x-fortran')
+
+        mt = reg.classify(data, filename="test.f95")
+        self.failUnlessEqual(str(mt), 'text/x-fortran')
+
+        mt = reg.classify(data, filename="makefile")
+        self.failUnlessEqual(str(mt), 'text/x-makefile')
+
+        mt = reg.classify(data, filename="Makefile")
+        self.failUnlessEqual(str(mt), 'text/x-makefile')
+
+        mt = reg.classify(data, filename="makefile.ac")
+        self.failUnlessEqual(str(mt), 'text/x-makefile')
+
+        mt = reg.classify(data, filename="makefile.in")
+        self.failUnlessEqual(str(mt), 'text/x-makefile')
+
+        mt = reg.classify(data, filename="AUTHORS")
+        self.failUnlessEqual(str(mt), 'text/x-authors')
+
+        mt = reg.classify(data, filename="INSTALL")
+        self.failUnlessEqual(str(mt), 'text/x-install')
+
+    def testLookup(self):
+        reg = self.registry
+        mt = reg.lookup('text/plain')
+        self.failUnless(isinstance(mt[0], text_plain), str(mt[0]))
+
+        # Test lookup of aliases in SMI database (see smi_mimetypes)
+        mt1 = reg.lookup('application/vnd.wordperfect')
+        mt2 = reg.lookup('application/wordperfect')
+        self.assertEqual(mt1, mt2)
+
+        mt = reg.lookup('text/notexistent')
+        self.failUnlessEqual(mt, ())
+
+    def testAdaptMt(self):
+        data, filename, mt = self.registry('bar', mimetype='text/xml')
+        # this test that data has been adaped and file seeked to 0
+        self.failUnlessEqual(data, 'bar')
+        self.failUnlessEqual(filename, None)
+        self.failUnless(isinstance(mt, text_xml), str(mt))
+
+    def testAdaptFile(self):
+        file = open(input_file_path("rest1.rst"))
+        data, filename, mt = self.registry(file)
+        # this test that data has been adaped and file seeked to 0
+        self.failUnlessEqual(data, file.read())
+        file.close()
+        self.failUnlessEqual(filename, "rest1.rst")
+        self.assertEqual(str(mt), 'text/x-rst')
+
+    def testAdaptData(self):
+        data, filename, mt = self.registry('<?xml ?>')
+        # this test that data has been adaped and file seeked to 0
+        self.failUnlessEqual(data, '<?xml ?>')
+        self.failUnlessEqual(filename, None)
+        self.failUnless(isinstance(mt, text_xml), str(mt))
+
+
+def test_suite():
+    from unittest import TestSuite, makeSuite
+    suite = TestSuite()
+    suite.addTest(makeSuite(TestMimeTypesclass))
+    return suite
+
+if __name__ == '__main__':
+    framework()

product/MimetypesRegistry/tests/utils.py

+import re
+import glob
+from unittest import TestSuite
+from sys import modules
+from os.path import join, abspath, dirname, basename
+
+def normalize_html(s):
+    s = re.sub(r"\s+", " ", s)
+    s = re.sub(r"(?s)\s+<", "<", s)
+    s = re.sub(r"(?s)>\s+", ">", s)
+    s = re.sub(r"\r", "", s)
+    return s
+
+PREFIX = abspath(dirname(__file__))
+
+def input_file_path(file):
+    return join(PREFIX, 'input', file)
+
+def output_file_path(file):
+    return join(PREFIX, 'output', file)
+
+def matching_inputs(pattern):
+    return [basename(path) for path in glob.glob(join(PREFIX, "input", pattern))]
+
+def load(dotted_name, globals=None):
+    """ load a python module from it's name """
+    mod = __import__(dotted_name, globals)
+    components = dotted_name.split('.')
+    for comp in components[1:]:
+        mod = getattr(mod, comp)
+    return mod

product/MimetypesRegistry/version.txt

+1.4.0-final
\ No newline at end of file

product/MimetypesRegistry/www/addMimeType.zpt

+<h1 tal:replace="structure here/manage_page_header|nothing">Header</h1>
+<h2 tal:define="manage_tabs_message options/manage_tabs_message | nothing"
+    tal:replace="structure here/manage_tabs">Tabs</h2>
+
+
+<form method="POST" action="manage_addMimeType"
+      tal:attributes="action string:${here/absolute_url}/manage_addMimeType;">
+  <div class="form-title">
+    Add a new mime type
+  </div>
+
+  <div tal:define="status python:request.get('portal_status', '')"
+       tal:condition="status"
+       class="error"
+       tal:content="status"
+       />
+
+  <table width="50%">
+    <tr>
+      <td class="form-label">Name</td>
+      <td class="form-element">
+        <input name="id"
+               tal:attributes="value python:request.get('id', '');"/>
+      </td>
+      </tr><tr>
+      <td class="form-label">Icon path</td>
+      <td class="form-element">
+        <input name="icon_path"
+               tal:attributes="value python:request.get('icon_path', '');"/>
+      </td>
+      </tr><tr>
+      <td class="form-label">Binary?
+      </td>
+      <td class="form-element">
+        <input name="binary" type="checkbox"
+               tal:attributes="value python:request.get('binary', '');"/>
+      </td>
+      </tr><tr>
+      <td class="form-label">Mime-types
+      </td>
+      <td class="form-element">
+        <textarea name="mimetypes:list"
+                  tal:content="python:request.get('mimetypes', '')"/>
+      </td>
+      </tr><tr>
+      <td class="form-label">Extensions
+      </td>
+      <td class="form-element">
+        <textarea name="extensions:list"
+                  tal:content="python:request.get('extensions', '')"/>
+      </td>
+    </tr><tr>
+      <td class="form-label">Globs
+      </td>
+      <td class="form-element">
+        <textarea name="globs:list"
+                  tal:content="python:request.get('globs', '')"/>
+      </td>
+    </tr>
+  </table>
+
+  <input type="submit" />
+</form>
+
+<tal:footer tal:replace="structure here/manage_page_footer|nothing">footer</tal:footer>

product/MimetypesRegistry/www/editMimeType.zpt

+<h1 tal:replace="structure here/manage_page_header|nothing">Header</h1>
+<h2 tal:define="manage_tabs_message options/manage_tabs_message | nothing"
+    tal:replace="structure here/manage_tabs">Tabs</h2>
+
+
+<form method="POST" action=""
+      tal:attributes="action string:${here/absolute_url}/manage_editMimeType;"
+      tal:define="mt python:here.lookup(request.form.get('mt_name'))[0]">
+
+  <div class="form-title">
+    Edit mime type <b tal:content="mt/name"/>.
+  </div>
+
+  <div tal:define="status python:request.get('portal_status', '')"
+       tal:condition="status"
+       class="error"
+       tal:content="status"
+       />
+
+  <input type="hidden" name="name" tal:attributes="value request/mt_name"/>
+
+  <table width="50%">
+    <tr>
+      <td>Name</td>
+      <td>
+        <input name="new_name"
+               tal:attributes="value python:request.get('new_name', mt.name());"/>
+      </td>
+      </tr><tr>
+      <td>Icon path</td>
+      <td>
+        <input name="icon_path"
+               tal:attributes="value python:request.get('icon_path', mt.icon_path);"/>
+      </td>
+      </tr><tr>
+      <td>Binary?
+      </td>
+      <td>
+        <input name="binary" type="checkbox"
+               tal:attributes="checked python:request.get('binary', mt.binary) and 1 or 0;"/>
+      </td>
+      </tr><tr>
+      <td>Mime-types
+      </td>
+      <td>
+        <textarea name="mimetypes"
+                  tal:content="python:request.get('mimetypes', '\n'.join(mt.mimetypes))"/>
+      </td>
+      </tr><tr>
+      <td>Extensions
+      </td>
+      <td>
+        <textarea name="extensions"
+                  tal:content="python:request.get('extensions', '\n'.join(mt.extensions))"/>
+      </td>
+      </tr><tr>
+      <td>Globs
+      </td>
+      <td>
+        <textarea name="globs"
+                  tal:content="python:request.get('globs', '\n'.join(mt.globs))"/>
+      </td>
+    </tr>
+  </table>
+
+  <input type="submit"/>
+</form>
+
+<tal:footer tal:replace="structure here/manage_page_footer|nothing">footer</tal:footer>