initial release of a utility to check the consistency of a ZCatalog

utilities/check_catalog.py

+#!/usr/bin/env python2.1
+
+##############################################################################
+# 
+# Zope Public License (ZPL) Version 1.0
+# -------------------------------------
+# 
+# Copyright (c) Digital Creations.  All rights reserved.
+# 
+# This license has been certified as Open Source(tm).
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+# 
+# 1. Redistributions in source code must retain the above copyright
+#    notice, this list of conditions, and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions, and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+# 
+# 3. Digital Creations requests that attribution be given to Zope
+#    in any manner possible. Zope includes a "Powered by Zope"
+#    button that is installed by default. While it is not a license
+#    violation to remove this button, it is requested that the
+#    attribution remain. A significant investment has been put
+#    into Zope, and this effort will continue if the Zope community
+#    continues to grow. This is one way to assure that growth.
+# 
+# 4. All advertising materials and documentation mentioning
+#    features derived from or use of this software must display
+#    the following acknowledgement:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    In the event that the product being advertised includes an
+#    intact Zope distribution (with copyright and license included)
+#    then this clause is waived.
+# 
+# 5. Names associated with Zope or Digital Creations must not be used to
+#    endorse or promote products derived from this software without
+#    prior written permission from Digital Creations.
+# 
+# 6. Modified redistributions of any form whatsoever must retain
+#    the following acknowledgment:
+# 
+#      "This product includes software developed by Digital Creations
+#      for use in the Z Object Publishing Environment
+#      (http://www.zope.org/)."
+# 
+#    Intact (re-)distributions of any official Zope release do not
+#    require an external acknowledgement.
+# 
+# 7. Modifications are encouraged but must be packaged separately as
+#    patches to official Zope releases.  Distributions that do not
+#    clearly separate the patches from the original work must be clearly
+#    labeled as unofficial distributions.  Modifications which do not
+#    carry the name Zope may be packaged in any form, as long as they
+#    conform to all of the clauses above.
+# 
+# 
+# Disclaimer
+# 
+#   THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS ``AS IS'' AND ANY
+#   EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+#   PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL CREATIONS OR ITS
+#   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+#   USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+#   ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+#   OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+#   OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+#   SUCH DAMAGE.
+# 
+# 
+# This software consists of contributions made by Digital Creations and
+# many individuals on behalf of Digital Creations.  Specific
+# attributions are listed in the accompanying credits file.
+# 
+##############################################################################
+
+""" script to consistency of a ZCatalog """
+
+__version__='$Revision: 1.1 $'[11:-2]
+
+
+import Zope    
+import os,sys,re,getopt
+from types import IntType
+from BTrees.IIBTree import IISet,difference,intersection
+
+def checkCatalog(path,indexes):
+    """ perform some consistency checks on a ZCatalog instance"""
+
+    root = Zope.app()
+
+    try:
+        catalog = root.unrestrictedTraverse(path)
+    except AttributeError:
+        print 'Error: catalog object not found'
+        sys.exit(1)
+
+    # get Catalog instance
+    _cat = catalog._catalog
+
+    # check Catalog internal BTrees
+    l_data  = list(_cat.data.keys())
+    l_data.sort()
+    l_uids  = list(_cat.uids.values())
+    l_uids.sort()
+    l_paths = list(_cat.data.keys())
+    l_paths.sort()
+
+    print "Checking catalog internal BTrees"
+    print "\tINFO: Mapping data:  %d entries" % len(l_data)
+    print "\tINFO: Mapping uids:  %d entries" % len(l_uids)
+    print "\tINFO: Mapping paths: %d entries" % len(l_paths)
+
+    if l_data == l_uids:
+        print "\tOK:  self.data==self.uids"
+    else:
+        print "\tERR: self.data!=self.uids"
+
+    if l_data == l_paths:
+        print "\tOK:  self.data==self.paths"
+    else:
+        print "\tERR: self.data!=self.paths"
+    
+
+    # check BTrees of indexes
+
+    for id,idx in _cat.indexes.items():
+
+        if indexes and not idx.meta_type in indexes: continue
+
+        print "Checking index '%s' (type: %s)" % (id, idx.meta_type)
+
+        if idx.meta_type in ['FieldIndex','KeywordIndex']:
+
+            # check forward entries
+            RIDS = IISet() 
+            for key, rids in idx._index.items():
+                if isinstance(rids,IntType): 
+                    RIDS.insert(  rids  )
+                else:
+                    map(RIDS.insert , rids.keys())
+
+            diff = difference(RIDS, IISet(_cat.data.keys()))
+            if len(diff)!=0:
+                print '\tERR: Problem with forward entries' 
+            else:    
+                print '\tOK:  Forward entries' 
+
+
+            # check backward entries
+            RIDS = IISet(idx._unindex.keys())
+            diff = difference(RIDS, IISet(_cat.data.keys()))
+            if len(diff)!=0:
+                print '\tERR: Problem with backward entries' 
+            else:
+                print '\tOK:  Backward entries' 
+
+        else:
+            print "\tWARN: no check implemented yet"
+        
+
+
+
+def usage():
+    print "Usage: %s [--FieldIndex] /path/to/ZCatalog" % \
+                os.path.basename(sys.argv[0])
+    print 
+    print "This scripts checks the consistency of the internal"
+    print "BTrees of a ZCatalog and its indexes."
+    sys.exit(1)
+    
+
+def main():
+
+    opts,args = getopt.getopt(sys.argv[1:],'h',['help','FieldIndex'])
+
+    indexes = []
+
+    for o,v in opts:
+        if o in ['-h','--help']: usage()
+        if o=='--FieldIndex':   indexes.append(o[2:])
+
+    checkCatalog(args,indexes)
+
+
+if __name__=='__main__':
+    main()