Refactored BTree specific code into separate template.

Updates Interfaces file with set operations and byValue method.

(re)Implemented new set operations and added byValue method.

Fixed bug in handling of empty BTreeItems.

src/BTrees/BTreeItemsTemplate.c

@@ -104,9 +104,9 @@ newBTreeItems(char kind,
 static void
 BTreeItems_dealloc(BTreeItems *self)
 {
-  Py_DECREF(self->firstbucket);
-  Py_DECREF(self->lastbucket);
-  Py_DECREF(self->currentbucket);
+  Py_XDECREF(self->firstbucket);
+  Py_XDECREF(self->lastbucket);
+  Py_XDECREF(self->currentbucket);
   PyMem_DEL(self);
 }
 
@@ -117,6 +117,8 @@ BTreeItems_length_or_nonzero(BTreeItems *self, int nonzero)
   Bucket *b, *next;
 
   b=self->firstbucket;
+  UNLESS(b) return 0;
+
   r=self->last + 1 - self->first;
 
   if (nonzero && r > 0) 
@@ -174,8 +176,14 @@ BTreeItems_seek(BTreeItems *self, int i)
   int delta, pseudoindex, currentoffset;
   Bucket *b, *currentbucket;
 
-  pseudoindex=self->pseudoindex;
   currentbucket=self->currentbucket;
+  UNLESS(currentbucket)
+    {
+      IndexError(i);
+      return -1;
+    }
+
+  pseudoindex=self->pseudoindex;
   Py_INCREF(currentbucket);
   currentoffset=self->currentoffset;
 
@@ -466,14 +474,88 @@ newBTreeItems(char kind,
   self->kind=kind;
   self->first=lowoffset;
   self->last=highoffset;
-  Py_INCREF(lowbucket);
+  Py_XINCREF(lowbucket);
   self->firstbucket = lowbucket;
-  Py_INCREF(highbucket);
+  Py_XINCREF(highbucket);
   self->lastbucket = highbucket;
-  Py_INCREF(lowbucket);
+  Py_XINCREF(lowbucket);
   self->currentbucket = lowbucket;
   self->currentoffset = lowoffset;
   self->pseudoindex = 0;
 
   return OBJECT(self);
 }
+
+static int 
+nextBTreeItems(SetIteration *i)
+{
+  if (i->position >= 0)
+    {
+      if (i->position)
+        {
+          DECREF_KEY(i->key);
+          DECREF_VALUE(i->value);
+        }
+      
+      if (BTreeItems_seek(ITEMS(i->set), i->position) >= 0)
+        {
+          Bucket *currentbucket;
+
+          currentbucket = BUCKET(ITEMS(i->set)->currentbucket);
+
+          UNLESS(PER_USE(currentbucket)) return -1;
+
+          COPY_KEY(i->key, currentbucket->keys[ITEMS(i->set)->currentoffset]);
+          INCREF_KEY(i->key);
+
+          COPY_VALUE(i->value, 
+                     currentbucket->values[ITEMS(i->set)->currentoffset]);
+          COPY_VALUE(i->value, 
+                   BUCKET(ITEMS(i->set)->currentbucket)
+                   ->values[ITEMS(i->set)->currentoffset]);
+          INCREF_VALUE(i->value);
+
+          i->position ++;
+
+          PER_ALLOW_DEACTIVATION(currentbucket);
+        }
+      else
+        {
+          i->position = -1;
+          PyErr_Clear();
+        }
+    }
+}
+
+static int 
+nextTreeSetItems(SetIteration *i)
+{
+  if (i->position >= 0)
+    {
+      if (i->position)
+        {
+          DECREF_KEY(i->key);
+        }
+      
+      if (BTreeItems_seek(ITEMS(i->set), i->position) >= 0)
+        {
+          Bucket *currentbucket;
+
+          currentbucket = BUCKET(ITEMS(i->set)->currentbucket);
+
+          UNLESS(PER_USE(currentbucket)) return -1;
+
+          COPY_KEY(i->key, currentbucket->keys[ITEMS(i->set)->currentoffset]);
+          INCREF_KEY(i->key);
+
+          i->position ++;
+
+          PER_ALLOW_DEACTIVATION(currentbucket);
+        }
+      else
+        {
+          i->position = -1;
+          PyErr_Clear();
+        }
+    }
+}

src/BTrees/BTreeTemplate.c

@@ -83,183 +83,6 @@
   
  ****************************************************************************/
 
-static char BTree_module_documentation[] = 
-""
-"\n$Id: BTreeTemplate.c,v 1.6 2001/02/16 20:49:11 jim Exp $"
-;
-
-#include "cPersistence.h"
-
-/***************************************************************
-   The following are macros that ought to be in cPersistence.h */
-#ifndef PER_USE 
-
-#define PER_USE(O) \
-(((O)->state != cPersistent_GHOST_STATE \
-  || (cPersistenceCAPI->setstate((PyObject*)(O)) >= 0)) \
- ? (((O)->state==cPersistent_UPTODATE_STATE) \
-    ? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0)
-
-#define Ghost_Test(O) ((O)->state == cPersistent_GHOST_STATE)
-#endif
-/***************************************************************/
-
-
-static void PyVar_Assign(PyObject **v, PyObject *e) { Py_XDECREF(*v); *v=e;}
-#define ASSIGN(V,E) PyVar_Assign(&(V),(E))
-#define ASSIGNC(V,E) (Py_INCREF((E)), PyVar_Assign(&(V),(E)))
-#define UNLESS(E) if (!(E))
-#define UNLESS_ASSIGN(V,E) ASSIGN(V,E); UNLESS(V)
-#define LIST(O) ((PyListObject*)(O))
-#define OBJECT(O) ((PyObject*)(O))
-
-#define MIN_BUCKET_ALLOC 16
-#define MAX_BTREE_SIZE(B) 256
-#define MAX_BUCKET_SIZE(B) DEFAULT_MAX_BUCKET_SIZE
-
-#define SameType_Check(O1, O2) ((O1)->ob_type==(O2)->ob_type)
-
-typedef struct BTreeItemStruct {
-  KEY_TYPE key;
-  PyObject *value;
-} BTreeItem;
-
-typedef struct Bucket_s {
-  cPersistent_HEAD
-  int size, len;
-  struct Bucket_s *next;
-  KEY_TYPE *keys;
-  VALUE_TYPE *values;
-} Bucket;
-
-#define BUCKET(O) ((Bucket*)(O))
-
-static void PyVar_AssignB(Bucket **v, Bucket *e) { Py_XDECREF(*v); *v=e;}
-#define ASSIGNB(V,E) PyVar_AssignB(&(V),(E))
-#define ASSIGNBC(V,E) (Py_INCREF((E)), PyVar_AssignB(&(V),(E)))
-
-typedef struct {
-  cPersistent_HEAD
-  int size, len;
-  Bucket *firstbucket;
-  BTreeItem *data;
-} BTree;
-
-#define BTREE(O) ((BTree*)(O))
-
-static PyObject *
-IndexError(int i)
-{                              
-  PyObject *v;
-
-  v=PyInt_FromLong(i);
-  UNLESS (v) {
-    v=Py_None;
-    Py_INCREF(v);
-  }
-  PyErr_SetObject(PyExc_IndexError, v);
-  Py_DECREF(v);
-  return NULL;
-}
-
-static Bucket *
-PreviousBucket(Bucket *current, Bucket *first, int i)
-{
-  if (! first) return NULL;
-  if (first==current)
-    {
-      IndexError(i);
-      return NULL;
-    }
-
-  Py_INCREF(first);
-  while (1)
-    {
-      PER_USE_OR_RETURN(first,NULL);
-      if (first->next==current) 
-        {
-          PER_ALLOW_DEACTIVATION(first);
-          return first;
-        }
-      else if (first->next)
-        {
-          Bucket *next = first->next;
-          Py_INCREF(next);
-          PER_ALLOW_DEACTIVATION(first);
-          Py_DECREF(first);
-          first=next;
-        }
-      else
-        {
-          PER_ALLOW_DEACTIVATION(first);
-          Py_DECREF(first);
-          IndexError(i);
-          return NULL;
-        }
-    }
-}
-
-static int 
-firstBucketOffset(Bucket **bucket, int *offset)
-{
-  Bucket *b;
-
-  *offset = (*bucket)->len - 1;
-  while ((*bucket)->len < 1)
-    {
-      b=(*bucket)->next;
-      if (b==NULL) return 0;
-      Py_INCREF(b);
-      PER_ALLOW_DEACTIVATION((*bucket));
-      ASSIGNB((*bucket), b);
-      UNLESS (PER_USE(*bucket)) return -1;
-      *offset = 0;
-    }
-}
-
-static int 
-lastBucketOffset(Bucket **bucket, int *offset, Bucket *firstbucket, int i)
-{
-  Bucket *b;
-
-  *offset = (*bucket)->len - 1;
-  while ((*bucket)->len < 1)
-    {
-      b=PreviousBucket((*bucket), firstbucket, i);
-      if (b==NULL) return 0;
-      PER_ALLOW_DEACTIVATION((*bucket));
-      ASSIGNB((*bucket), b);
-      UNLESS (PER_USE(*bucket)) return -1;
-      *offset = (*bucket)->len - 1;
-    }
-}
-
-static void *
-PyMalloc(size_t sz)
-{
-  void *r;
-
-  if (r=malloc(sz)) return r;
-
-  PyErr_NoMemory();
-  return NULL;
-}
-
-static void *
-PyRealloc(void *p, size_t sz)
-{
-  void *r;
-
-  if (r=realloc(p,sz)) return r;
-
-  PyErr_NoMemory();
-  return NULL;
-}
-
-#include "BTreeItemsTemplate.c"
-#include "BucketTemplate.c"
-#include "SetTemplate.c"
-
 /*
 ** _BTree_get
 **
@@ -1119,7 +942,7 @@ BTree_rangeSearch(BTree *self, PyObject *args, char type)
 
  empty:
   PER_ALLOW_DEACTIVATION(self);
-  return PyTuple_New(0);
+  return newBTreeItems(type, 0, 0, 0, 0);
 }
 
 /*
@@ -1149,6 +972,73 @@ BTree_items(BTree *self, PyObject *args)
   return BTree_rangeSearch(self,args,'i');
 }
 
+static PyObject *
+BTree_byValue(BTree *self, PyObject *args)
+{
+  PyObject *r=0, *o=0, *item=0, *omin;
+  VALUE_TYPE min;
+  VALUE_TYPE v;
+  int i, l, copied=1;
+  SetIteration it={0,0};
+
+  PER_USE_OR_RETURN(self, NULL);
+
+  UNLESS (PyArg_ParseTuple(args, "O", &omin)) return NULL;
+  COPY_VALUE_FROM_ARG(min, omin, &copied);
+  UNLESS(copied) return NULL;
+    
+  UNLESS (r=PyList_New(0)) goto err;
+
+  it.set=BTree_rangeSearch(self, NULL, 'i');
+  UNLESS(it.set) goto err;
+
+  if (nextBTreeItems(&it) < 0) goto err;
+
+  while (it.position >= 0)
+    {
+      if (TEST_VALUE(it.value, min) >= 0)
+        {      
+          UNLESS (item = PyTuple_New(2)) goto err;
+
+          COPY_KEY_TO_OBJECT(o, it.key);
+          UNLESS (o) goto err;
+          PyTuple_SET_ITEM(item, 1, o);
+
+          COPY_VALUE(v, it.value);
+          NORMALIZE_VALUE(v, min);
+          COPY_VALUE_TO_OBJECT(o, v);
+          DECREF_VALUE(v);
+          UNLESS (o) goto err;
+          PyTuple_SET_ITEM(item, 0, o);
+      
+          if (PyList_Append(r, item) < 0) goto err;
+          Py_DECREF(item);
+          item = 0;
+        }
+      if (nextBTreeItems(&it) < 0) goto err;
+    }
+
+  item=PyObject_GetAttr(r,sort_str);
+  UNLESS (item) goto err;
+  ASSIGN(item, PyObject_CallObject(item, NULL));
+  UNLESS (item) goto err;
+  ASSIGN(item, PyObject_GetAttr(r, reverse_str));
+  UNLESS (item) goto err;
+  ASSIGN(item, PyObject_CallObject(item, NULL));
+  UNLESS (item) goto err;
+  Py_DECREF(item);
+
+  PER_ALLOW_DEACTIVATION(self);
+  return r;
+
+ err:
+  PER_ALLOW_DEACTIVATION(self);
+  Py_XDECREF(r);
+  Py_XDECREF(it.set);
+  Py_XDECREF(item);
+  return NULL;
+}
+
 /*
 ** BTree_getm
 */
@@ -1196,11 +1086,15 @@ static struct PyMethodDef BTree_methods[] = {
   {"has_key",	(PyCFunction) BTree_has_key,	METH_VARARGS,
      "has_key(key) -- Test whether the bucket contains the given key"},
   {"keys",	(PyCFunction) BTree_keys,	METH_VARARGS,
-     "keys() -- Return the keys"},
+     "keys([min, max]) -- Return the keys"},
   {"values",	(PyCFunction) BTree_values,	METH_VARARGS,
-     "values() -- Return the values"},
+     "values([min, max]) -- Return the values"},
   {"items",	(PyCFunction) BTree_items,	METH_VARARGS,
-     "items() -- Return the items"},
+     "items([min, max]) -- Return the items"},
+  {"byValue",	(PyCFunction) BTree_byValue,	METH_VARARGS,
+   "byValue(min) -- "
+   "Return value-keys with values >= min and reverse sorted by values"
+  },
   {"get",	(PyCFunction) BTree_getm,	METH_VARARGS,
    "get(key[,default]) -- Look up a value\n\n"
    "Return the default (or None) if the key is not found."
@@ -1324,76 +1218,3 @@ static PyExtensionClass BTreeType = {
   EXTENSIONCLASS_BASICNEW_FLAG | PERSISTENT_TYPE_FLAG 
   | EXTENSIONCLASS_NOINSTDICT_FLAG,
 };
-
-#include "TreeSetTemplate.c"
-
-#include "SetOpTemplate.c"
-
-static struct PyMethodDef module_methods[] = {
-  {"union", (PyCFunction) union_m,	METH_VARARGS,
-   "union(o1, o2 [, w1, w2]) -- compurte the union of o1 and o2\n"
-   "\nw1 and w2 are weights, which are only meaningful for integer values"
-  },
-  {"intersection", (PyCFunction) intersection_m,	METH_VARARGS,
-   "intersection(o1, o2 [, w1, w2]) -- "
-   "compurte the intersection of o1 and o2\n"
-   "\nw1 and w2 are weights, which are only meaningful for integer values"
-  },
-  {"difference", (PyCFunction) difference_m,	METH_VARARGS,
-   "difference(o1, o2 [, w1, w2]) -- "
-   "compurte the difference between o1 and o2\n"
-   "\nw1 and w2 are weights, which are only meaningful for integer values"
-  },
-  {NULL,		NULL}		/* sentinel */
-};
-
-void 
-INITMODULE ()
-{
-  PyObject *m, *d;
-
-  UNLESS (PyExtensionClassCAPI=PyCObject_Import("ExtensionClass","CAPI"))
-      return;
-
-  if (cPersistenceCAPI=PyCObject_Import("cPersistence","CAPI"))
-    {
-	BucketType.methods.link=cPersistenceCAPI->methods;
-	BucketType.tp_getattro=cPersistenceCAPI->getattro;
-	BucketType.tp_setattro=cPersistenceCAPI->setattro;
-
-	SetType.methods.link=cPersistenceCAPI->methods;
-	SetType.tp_getattro=cPersistenceCAPI->getattro;
-	SetType.tp_setattro=cPersistenceCAPI->setattro;
-
-	BTreeType.methods.link=cPersistenceCAPI->methods;
-	BTreeType.tp_getattro=cPersistenceCAPI->getattro;
-	BTreeType.tp_setattro=cPersistenceCAPI->setattro;
-
-	TreeSetType.methods.link=cPersistenceCAPI->methods;
-	TreeSetType.tp_getattro=cPersistenceCAPI->getattro;
-	TreeSetType.tp_setattro=cPersistenceCAPI->setattro;
-    }
-  else return;
-
-  BTreeItemsType.ob_type=&PyType_Type;
-
-  /* Create the module and add the functions */
-  m = Py_InitModule4(PREFIX "BTree", module_methods,
-		     BTree_module_documentation,
-		     (PyObject*)NULL,PYTHON_API_VERSION);
-
-  /* Add some symbolic constants to the module */
-  d = PyModule_GetDict(m);
-
-  PyDict_SetItemString(d, "__version__",
-		       PyString_FromString("$Revision: 1.6 $"));
-
-  PyExtensionClass_Export(d,PREFIX "Bucket", BucketType);
-  PyExtensionClass_Export(d,PREFIX "BTree", BTreeType);
-  PyExtensionClass_Export(d,PREFIX "Set", SetType);
-  PyExtensionClass_Export(d,PREFIX "TreeSet", TreeSetType);
- 
-  /* Check for errors */
-  if (PyErr_Occurred())
-    Py_FatalError("can't initialize module " PREFIX "BTree");
-}

src/BTrees/BucketTemplate.c

@@ -663,6 +663,69 @@ bucket_items(Bucket *self, PyObject *args)
   return NULL;
 }
 
+static PyObject *
+bucket_byValue(Bucket *self, PyObject *args)
+{
+  PyObject *r=0, *o=0, *item=0, *omin;
+  VALUE_TYPE min;
+  VALUE_TYPE v;
+  int i, l, copied=1;
+
+  PER_USE_OR_RETURN(self, NULL);
+
+  UNLESS (PyArg_ParseTuple(args, "O", &omin)) return NULL;
+  COPY_VALUE_FROM_ARG(min, omin, &copied);
+  UNLESS(copied) return NULL;
+
+  for (i=0, l=0; i < self->len; i++) 
+    if (TEST_VALUE(self->values[i], min) >= 0) 
+      l++;
+    
+  UNLESS (r=PyList_New(l)) goto err;
+
+  for (i=0, l=0; i < self->len; i++)
+    {
+      if (TEST_VALUE(self->values[i], min) < 0) continue;
+      
+      UNLESS (item = PyTuple_New(2)) goto err;
+
+      COPY_KEY_TO_OBJECT(o, self->keys[i]);
+      UNLESS (o) goto err;
+      PyTuple_SET_ITEM(item, 1, o);
+
+      COPY_VALUE(v, self->values[i]);
+      NORMALIZE_VALUE(v, min);
+      COPY_VALUE_TO_OBJECT(o, v);
+      DECREF_VALUE(v);
+      UNLESS (o) goto err;
+      PyTuple_SET_ITEM(item, 0, o);
+      
+      if (PyList_SetItem(r, l, item) < 0) goto err;
+      l++;
+
+      item = 0;
+    }
+
+  item=PyObject_GetAttr(r,sort_str);
+  UNLESS (item) goto err;
+  ASSIGN(item, PyObject_CallObject(item, NULL));
+  UNLESS (item) goto err;
+  ASSIGN(item, PyObject_GetAttr(r, reverse_str));
+  UNLESS (item) goto err;
+  ASSIGN(item, PyObject_CallObject(item, NULL));
+  UNLESS (item) goto err;
+  Py_DECREF(item);
+
+  PER_ALLOW_DEACTIVATION(self);
+  return r;
+
+ err:
+  PER_ALLOW_DEACTIVATION(self);
+  Py_XDECREF(r);
+  Py_XDECREF(item);
+  return NULL;
+}
+
 /*
 ** bucket__p_deactivate
 **
@@ -894,7 +957,7 @@ static struct PyMethodDef Bucket_methods[] = {
   {"__setstate__", (PyCFunction) bucket_setstate,	METH_VARARGS,
    "__setstate__() -- Set the state of the object"},
   {"keys",	(PyCFunction) bucket_keys,	METH_VARARGS,
-     "keys() -- Return the keys"},
+     "keys([min, max]) -- Return the keys"},
   {"has_key",	(PyCFunction) bucket_has_key,	METH_VARARGS,
      "has_key(key) -- Test whether the bucket contains the given key"},
   {"clear",	(PyCFunction) bucket_clear,	METH_VARARGS,
@@ -906,9 +969,13 @@ static struct PyMethodDef Bucket_methods[] = {
    "minKey([key]) -- Fine the minimum key\n\n"
    "If an argument is given, find the minimum >= the argument"},
   {"values",	(PyCFunction) bucket_values,	METH_VARARGS,
-     "values() -- Return the values"},
+     "values([min, max]) -- Return the values"},
   {"items",	(PyCFunction) bucket_items,	METH_VARARGS,
-     "items() -- Return the items"},
+     "items([min, max])) -- Return the items"},
+  {"byValue",	(PyCFunction) bucket_byValue,	METH_VARARGS,
+   "byValue(min) -- "
+   "Return value-keys with values >= min and reverse sorted by values"
+  },
   {"get",	(PyCFunction) bucket_getm,	METH_VARARGS,
    "get(key[,default]) -- Look up a value\n\n"
    "Return the default (or None) if the key is not found."
@@ -1001,3 +1068,34 @@ static PyExtensionClass BucketType = {
   EXTENSIONCLASS_BASICNEW_FLAG | PERSISTENT_TYPE_FLAG 
   | EXTENSIONCLASS_NOINSTDICT_FLAG,
 };
+
+
+static int 
+nextBucket(SetIteration *i)
+{
+  UNLESS(PER_USE(BUCKET(i->set))) return -1;
+          
+  if (i->position >= 0)
+    {
+      if (i->position)
+        {
+          DECREF_KEY(i->key);
+          DECREF_VALUE(i->value);
+        }
+
+      if (i->position < BUCKET(i->set)->len)
+        {
+          COPY_KEY(i->key, BUCKET(i->set)->keys[i->position]);
+          INCREF_KEY(i->key);
+          COPY_VALUE(i->value, BUCKET(i->set)->values[i->position]);
+          INCREF_VALUE(i->value);
+          i->position ++;
+        }
+      else
+        i->position = -1;
+    }
+
+  PER_ALLOW_DEACTIVATION(BUCKET(i->set));
+          
+  return 0;
+}

src/BTrees/Interfaces.py

@@ -147,6 +147,14 @@ class IDictionaryIsh(IKeyed, ISized):
         items having keys less than or equal to the given min.
         A max value of None is ignored.
         """
+
+    def byValue(minValue):
+        """Return a sequence of value-key pairs, sorted by value
+
+        Values < min are ommitted and other values are "normalized" by
+        the minimum value. This normalization may be a noop, but, for
+        integer values, the normalization is division.
+        """
     
 class IBTree(IDictionaryIsh):
 
@@ -168,6 +176,124 @@ class IBTree(IDictionaryIsh):
           while not t.insert(key, value):
               key=generate_key()
         """
+
+class IMerge(Interfaces.Base):
+    """Object with methods for merging sets, buckets, and trees.
+
+    These methods are supplied in modules that define collection
+    classes with particular key and value types. The operations apply
+    only to collections from the same module.  For example, the
+    IIBTree.union can only be used with IIBTree.IIBTree,
+    IIBTree.IIBucket, IIBTree.IISet, and IIBTree.IITreeSet.
+
+    The implementing module has a value type. The IOBTree and OOBTree
+    modules have object value type. The IIBTree and OIBTree modules
+    have integer value tyoes. Other modules may be defined in the
+    future that have other value types.
+
+    The individual types are classified into set (Set and TreeSet) and
+    mapping (Bucket and BTree) types.
+    """
+
+    def difference(c1, c2):
+        """Return the keys or items in c1 for which there is no key in
+        c2.
+
+        If c1 is None, then None is returned.  If c2 is none, then c1
+        is returned.
+        """
+
+    def union(c1, c2):
+        """Compute the Union of c1 and c2.
+
+        If c1 is None, then c2 is returned, otherwise, if c2 is None,
+        then c1 is returned.
+
+        The output is a Set containing keys from the input
+        collections.
+        """
+
+    def intersection(c1, c2):
+        """Compute the Union of c1 and c2.
+
+        If c1 is None, then c2 is returned, otherwise, if c2 is None,
+        then c1 is returned.
+
+        The output is a Set containing matching keys from the input
+        collections.
+        """
+
+class IIMerge(IMerge):
+    """Merge collections with integer value type.
+
+    A primary intent is to support operations with no or integer
+    values, which are used as "scores" to rate indiviual keys. That
+    is, in this context, a BTree or Bucket is viewed as a set with
+    scored keys, using integer scores.
+    """
+
+    def weightedUnion(c1, c2, weight1=1, weight2=1):
+        """Compute the weighted Union of c1 and c2.
+
+        If c1 and c2 are None, the output is 0 and None
+
+        if c1 is None and c2 is not None, the output is weight2 and
+        c2.
+
+        if c1 is not None and c2 not None, the output is weight1 and
+        c1.
+
+        If c1 and c2 are not None, the output is 1 and a Bucket
+        such that the output values are::
+
+          v1*weight1 + v2*weight2
+
+          where:
+
+            v1 is 0 if the key was not in c1. Otherwise, v1 is 1, if
+            c1 is a set, or the value from c1.
+
+            v2 is 0 if the key was not in c2. Otherwise, v2 is 2, if
+            c2 is a set, or the value from c2.
+
+        Note that c1 and c2 must be collections. None may not be
+        passed as one of the collections.
+        """
+
+    def weightedIntersection(c1, c2, weight1=1, weight2=1):
+        """Compute the weighted intersection of c1 and c2.
+
+        If c1 and c2 are None, the output is None, None.
+
+        if c1 is None and c2 is not None, the output is weight2 and
+        c2.
+
+        if c1 is not None and c2 not None, the output is weight1 and
+        c1.
+
+        If c1 and c2 are sets, the output is the sum of the weights
+        and the (unweighted) intersection of the sets.
+
+        If c1 and c2 are not None and not both sets, the output is 1
+        and a Bucket such that the output values are::
+
+          v1*weight1 + v2*weight2
+
+          where:
+
+            v1 is 0 if the key was not in c1. Otherwise, v1 is 1, if
+            c1 is a set, or the value from c1.
+
+            v2 is 0 if the key was not in c2. Otherwise, v2 is 2, if
+            c2 is a set, or the value from c2.
+
+        Note that c1 and c2 must be collections. None may not be
+        passed as one of the collections.
+        """
+
+        
+    
+        
     
 Interface.assertTypeImplements(OOBTree.OOSet, ISet)
 Interface.assertTypeImplements(OOBTree.OOTreeSet, ITreeSet)

src/BTrees/SetOpTemplate.c

@@ -87,147 +87,6 @@
  Set operations
  ****************************************************************************/
 
-typedef struct SetIteration_s 
-{
-  PyObject *set;
-  int position;
-  int hasValue;
-  KEY_TYPE key;
-  VALUE_TYPE value;
-  int (*next)(struct SetIteration_s*);
-} SetIteration;
-
-static int 
-nextBucket(SetIteration *i)
-{
-  UNLESS(PER_USE(BUCKET(i->set))) return -1;
-          
-  if (i->position >= 0)
-    {
-      if (i->position)
-        {
-          DECREF_KEY(i->key);
-          DECREF_VALUE(i->value);
-        }
-
-      if (i->position < BUCKET(i->set)->len)
-        {
-          COPY_KEY(i->key, BUCKET(i->set)->keys[i->position]);
-          INCREF_KEY(i->key);
-          COPY_VALUE(i->value, BUCKET(i->set)->values[i->position]);
-          INCREF_VALUE(i->value);
-          i->position ++;
-        }
-      else
-        i->position = -1;
-    }
-
-  PER_ALLOW_DEACTIVATION(BUCKET(i->set));
-          
-  return 0;
-}
-
-static int 
-nextSet(SetIteration *i)
-{
-  UNLESS(PER_USE(BUCKET(i->set))) return -1;
-          
-  if (i->position >= 0)
-    {
-      if (i->position)
-        {
-          DECREF_KEY(i->key);
-        }
-
-      if (i->position < BUCKET(i->set)->len)
-        {
-          COPY_KEY(i->key, BUCKET(i->set)->keys[i->position]);
-          INCREF_KEY(i->key);
-          i->position ++;
-        }
-      else
-        i->position = -1;
-    }
-
-  PER_ALLOW_DEACTIVATION(BUCKET(i->set));
-          
-  return 0;
-}
-
-static int 
-nextBTreeItems(SetIteration *i)
-{
-  if (i->position >= 0)
-    {
-      if (i->position)
-        {
-          DECREF_KEY(i->key);
-          DECREF_VALUE(i->value);
-        }
-      
-      if (BTreeItems_seek(ITEMS(i->set), i->position) >= 0)
-        {
-          Bucket *currentbucket;
-
-          currentbucket = BUCKET(ITEMS(i->set)->currentbucket);
-
-          UNLESS(PER_USE(currentbucket)) return -1;
-
-          COPY_KEY(i->key, currentbucket->keys[ITEMS(i->set)->currentoffset]);
-          INCREF_KEY(i->key);
-
-          COPY_VALUE(i->value, 
-                     currentbucket->values[ITEMS(i->set)->currentoffset]);
-          COPY_VALUE(i->value, 
-                   BUCKET(ITEMS(i->set)->currentbucket)
-                   ->values[ITEMS(i->set)->currentoffset]);
-          INCREF_VALUE(i->value);
-
-          i->position ++;
-
-          PER_ALLOW_DEACTIVATION(currentbucket);
-        }
-      else
-        {
-          i->position = -1;
-          PyErr_Clear();
-        }
-    }
-}
-
-static int 
-nextTreeSetItems(SetIteration *i)
-{
-  if (i->position >= 0)
-    {
-      if (i->position)
-        {
-          DECREF_KEY(i->key);
-        }
-      
-      if (BTreeItems_seek(ITEMS(i->set), i->position) >= 0)
-        {
-          Bucket *currentbucket;
-
-          currentbucket = BUCKET(ITEMS(i->set)->currentbucket);
-
-          UNLESS(PER_USE(currentbucket)) return -1;
-
-          COPY_KEY(i->key, currentbucket->keys[ITEMS(i->set)->currentoffset]);
-          INCREF_KEY(i->key);
-
-          i->position ++;
-
-          PER_ALLOW_DEACTIVATION(currentbucket);
-        }
-      else
-        {
-          i->position = -1;
-          PyErr_Clear();
-        }
-    }
-}
-
 static int
 initSetIteration(SetIteration *i, PyObject *s, int w, int *merge)
 {
@@ -238,8 +97,14 @@ initSetIteration(SetIteration *i, PyObject *s, int w, int *merge)
       i->set = s;
       Py_INCREF(s);
 
-      if (w > 0) *merge=1;
-      i->next=nextBucket;
+      if (w >= 0) 
+        {
+          *merge=1;
+          i->next=nextBucket;
+        }
+      else
+        i->next=nextSet;
+
       i->hasValue=1;
     }
   else if (ExtensionClassSubclassInstance_Check(s, &BTreeType))
@@ -247,8 +112,13 @@ initSetIteration(SetIteration *i, PyObject *s, int w, int *merge)
       i->set=BTree_rangeSearch(BTREE(s), NULL, 'i');
       UNLESS(i->set) return -1;
 
-      if (w > 0) *merge=1;
-      i->next=nextBTreeItems;
+      if (w >= 0) 
+        {
+          *merge=1;
+          i->next=nextBTreeItems;
+        }
+      else
+        i->next=nextTreeSetItems;
       i->hasValue=1;
     }
   else if (ExtensionClassSubclassInstance_Check(s, &SetType))
@@ -256,7 +126,6 @@ initSetIteration(SetIteration *i, PyObject *s, int w, int *merge)
       i->set = s;
       Py_INCREF(s);
 
-      if (w > 1) *merge=1;
       i->next=nextSet;
       i->hasValue=0;
     }
@@ -265,7 +134,6 @@ initSetIteration(SetIteration *i, PyObject *s, int w, int *merge)
       i->set=BTree_rangeSearch(BTREE(s), NULL, 'k');
       UNLESS(i->set) return -1;
 
-      if (w > 1) *merge=1;
       i->next=nextTreeSetItems;
       i->hasValue=0;
     }
@@ -312,27 +180,6 @@ set_operation(PyObject *s1, PyObject *s2,
   SetIteration i1 = {0,0,0}, i2 = {0,0,0};
   int cmp, merge=0;
 
-  if (s1==Py_None)
-    {
-      if (c12 || c2)
-        {
-          Py_INCREF(s2);
-          return s2;
-        }
-      Py_INCREF(s1);
-      return s1;
-    }
-  else if (s2==Py_None)
-    {
-      if (c12 || c1)
-        {
-          Py_INCREF(s1);
-          return s1;
-        }
-      Py_INCREF(s2);
-      return s2;
-    }
-
   if (initSetIteration(&i1, s1, w1, &merge) < 0) return NULL;
   if (initSetIteration(&i2, s2, w2, &merge) < 0) return NULL;
 
@@ -450,35 +297,106 @@ err:
   return NULL;
 }
 
+static PyObject *
+difference_m(PyObject *ignored, PyObject *args)
+{
+  PyObject *o1, *o2;
+
+  UNLESS(PyArg_ParseTuple(args, "OO", &o1, &o2)) return NULL;
+
+
+  if (o1==Py_None || o2==Py_None) 
+    {
+      Py_INCREF(o1);
+      return Py_None;
+    }
+  
+  return set_operation(o1, o2, 1, -1, 1, 0, 0);
+}         
+
 static PyObject *
 union_m(PyObject *ignored, PyObject *args)
 {
   PyObject *o1, *o2;
-  int w1=1, w2=1;
 
-  UNLESS(PyArg_ParseTuple(args, "OO|ii", &o1, &o2, &w1, &w2)) return NULL;
+  UNLESS(PyArg_ParseTuple(args, "OO", &o1, &o2)) return NULL;
+
+  if (o1==Py_None)
+    {
+      Py_INCREF(o2);
+      return o2;
+    }
+  else if (o2==Py_None)
+    {
+      Py_INCREF(o1);
+      return o1;
+    }
   
-  return set_operation(o1, o2, w1, w2, 1, 1, 1);
+  return set_operation(o1, o2, -1, -1, 1, 1, 1);
 }         
 
 static PyObject *
 intersection_m(PyObject *ignored, PyObject *args)
+{
+  PyObject *o1, *o2;
+
+  UNLESS(PyArg_ParseTuple(args, "OO", &o1, &o2)) return NULL;
+
+  if (o1==Py_None)
+    {
+      Py_INCREF(o2);
+      return o2;
+    }
+  else if (o2==Py_None)
+    {
+      Py_INCREF(o1);
+      return o1;
+    }
+  
+  return set_operation(o1, o2, -1, -1, 0, 1, 0);
+}         
+
+#ifdef MERGE
+
+static PyObject *
+wunion_m(PyObject *ignored, PyObject *args)
 {
   PyObject *o1, *o2;
   int w1=1, w2=1;
 
   UNLESS(PyArg_ParseTuple(args, "OO|ii", &o1, &o2, &w1, &w2)) return NULL;
+
+  if (o1==Py_None)
+    return Py_BuildValue("iO", (o2==Py_None ? 0 : w2), o2);
+  else if (o2==Py_None)
+    return Py_BuildValue("iO", w1, o1);
   
-  return set_operation(o1, o2, w1, w2, 0, 1, 0);
+  o1=set_operation(o1, o2, w1, w2, 1, 1, 1);
+  if (o1) ASSIGN(o1, Py_BuildValue("iO", 1, o1));
+
+  return o1;
 }         
 
 static PyObject *
-difference_m(PyObject *ignored, PyObject *args)
+wintersection_m(PyObject *ignored, PyObject *args)
 {
   PyObject *o1, *o2;
   int w1=1, w2=1;
 
   UNLESS(PyArg_ParseTuple(args, "OO|ii", &o1, &o2, &w1, &w2)) return NULL;
+
+  if (o1==Py_None)
+    return Py_BuildValue("iO", (o2==Py_None ? 0 : w2), o2);
+  else if (o2==Py_None)
+    return Py_BuildValue("iO", w1, o1);
   
-  return set_operation(o1, o2, w1, w2, 1, 0, 0);
+  o1=set_operation(o1, o2, w1, w2, 0, 1, 0);
+  if (o1) 
+    ASSIGN(o1, Py_BuildValue("iO", 
+            ((o1->ob_type == (PyTypeObject*)(&SetType)) ? w2+w1 : 1),
+                             o1));
+
+  return o1;
 }         
+
+#endif

src/BTrees/SetTemplate.c

@@ -314,3 +314,30 @@ static PyExtensionClass SetType = {
   EXTENSIONCLASS_BASICNEW_FLAG | PERSISTENT_TYPE_FLAG 
   | EXTENSIONCLASS_NOINSTDICT_FLAG,
 };
+
+static int 
+nextSet(SetIteration *i)
+{
+  UNLESS(PER_USE(BUCKET(i->set))) return -1;
+          
+  if (i->position >= 0)
+    {
+      if (i->position)
+        {
+          DECREF_KEY(i->key);
+        }
+
+      if (i->position < BUCKET(i->set)->len)
+        {
+          COPY_KEY(i->key, BUCKET(i->set)->keys[i->position]);
+          INCREF_KEY(i->key);
+          i->position ++;
+        }
+      else
+        i->position = -1;
+    }
+
+  PER_ALLOW_DEACTIVATION(BUCKET(i->set));
+          
+  return 0;
+}

src/BTrees/intkeymacros.h

 #define KEY_TYPE int
-#define KET_PARSE "i"
 #define TEST_KEY(KEY, TARGET) ( (KEY) - (TARGET) )
 #define DECREF_KEY(KEY)
 #define INCREF_KEY(k)

src/BTrees/intvaluemacros.h

 #define VALUE_TYPE int
+#define TEST_VALUE(VALUE, TARGET) ( (VALUE) - (TARGET) )
 #define DECLARE_VALUE(NAME) VALUE_TYPE NAME
 #define VALUE_PARSE "i"
 #define DECREF_VALUE(k)
@@ -10,6 +11,8 @@
       PyErr_SetString(PyExc_TypeError, "expected integer value"); \
       *(STATUS)=0; } 
   
+#define NORMALIZE_VALUE(V, MIN) ((MIN) > 0) ? ((V)/=(MIN)) : 0
+
 #define MERGE_DEFAULT 1
 #define MERGE(O1, w1, O2, w2) ((O1)*(w1)+(O2)*(w2))
 #define MERGE_WEIGHT(O, w) ((O)*(w))

src/BTrees/objectvaluemacros.h

 #define VALUE_TYPE PyObject *
+#define TEST_VALUE(VALUE, TARGET) PyObject_Compare((VALUE),(TARGET))
 #define DECLARE_VALUE(NAME) VALUE_TYPE NAME
 #define INCREF_VALUE(k) Py_INCREF(k)
 #define DECREF_VALUE(k) Py_DECREF(k)
 #define COPY_VALUE(k,e) k=(e)
 #define COPY_VALUE_TO_OBJECT(O, K) O=(K); Py_INCREF(O)
 #define COPY_VALUE_FROM_ARG(TARGET, ARG, S) TARGET=(ARG)
-
+#define NORMALIZE_VALUE(V, MIN) Py_INCREF(V)