Skip to content

Z
ZEO
Commit 226ee421 authored by Toby Dickenson's avatar Toby Dickenson
Browse files
Options

merged toby-stiff-cache-branch and toby-unicode-branch

parent 87c6d4a6
Hide whitespace changes
Inline Side-by-side
Showing with 2669 additions and 1429 deletions
src/BTrees/BTreeTemplate.c
View file @ 226ee421
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
****************************************************************************/ ****************************************************************************/
#define BTREETEMPLATE_C "$Id: BTreeTemplate.c,v 1.25 2002/03/08 18:33:01 jeremy Exp $\n" #define BTREETEMPLATE_C "$Id: BTreeTemplate.c,v 1.26 2002/03/27 10:14:01 htrd Exp $\n"
/* /*
** _BTree_get ** _BTree_get
...@@ -540,7 +540,7 @@ _BTree_clear(BTree *self) ...@@ -540,7 +540,7 @@ _BTree_clear(BTree *self)
if (self->firstbucket) if (self->firstbucket)
{ {
ASSERT(self->firstbucket->ob_refcnt > 1, ASSERT(self->firstbucket->ob_refcnt > 0,
"Invalid firstbucket pointer", -1); "Invalid firstbucket pointer", -1);
Py_DECREF(self->firstbucket); Py_DECREF(self->firstbucket);
self->firstbucket=NULL; self->firstbucket=NULL;
...@@ -573,7 +573,7 @@ BTree__p_deactivate(BTree *self, PyObject *args) ...@@ -573,7 +573,7 @@ BTree__p_deactivate(BTree *self, PyObject *args)
if (self->state==cPersistent_UPTODATE_STATE && self->jar) if (self->state==cPersistent_UPTODATE_STATE && self->jar)
{ {
if (_BTree_clear(self) < 0) return NULL; if (_BTree_clear(self) < 0) return NULL;
self->state=cPersistent_GHOST_STATE; PER_GHOSTIFY(self);
} }
Py_INCREF(Py_None); Py_INCREF(Py_None);
......
src/BTrees/BucketTemplate.c
View file @ 226ee421
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
****************************************************************************/ ****************************************************************************/
#define BUCKETTEMPLATE_C "$Id: BucketTemplate.c,v 1.28 2002/03/08 18:33:01 jeremy Exp $\n" #define BUCKETTEMPLATE_C "$Id: BucketTemplate.c,v 1.29 2002/03/27 10:14:01 htrd Exp $\n"
/* /*
** _bucket_get ** _bucket_get
...@@ -809,7 +809,7 @@ bucket__p_deactivate(Bucket *self, PyObject *args) ...@@ -809,7 +809,7 @@ bucket__p_deactivate(Bucket *self, PyObject *args)
if (self->state==cPersistent_UPTODATE_STATE && self->jar) if (self->state==cPersistent_UPTODATE_STATE && self->jar)
{ {
if (_bucket_clear(self) < 0) return NULL; if (_bucket_clear(self) < 0) return NULL;
self->state=cPersistent_GHOST_STATE; PER_GHOSTIFY(self);
} }
Py_INCREF(Py_None); Py_INCREF(Py_None);
......
src/Persistence/cPersistence.c
View file @ 226ee421
...@@ -14,11 +14,21 @@ ...@@ -14,11 +14,21 @@
static char cPersistence_doc_string[] = static char cPersistence_doc_string[] =
"Defines Persistent mixin class for persistent objects.\n" "Defines Persistent mixin class for persistent objects.\n"
"\n" "\n"
"$Id: cPersistence.c,v 1.50 2002/03/08 18:36:13 jeremy Exp $\n"; "$Id: cPersistence.c,v 1.51 2002/03/27 10:14:04 htrd Exp $\n";
#include <string.h> #include <string.h>
#include "cPersistence.h" #include "cPersistence.h"
/* the layout of this struct is the same as the start of ccobject in cPickleCache.c */
struct ccobject_head_struct {
PyObject_HEAD
CPersistentRing ring_home;
int non_ghost_count;
};
#define HOME(O) ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->ring_home)) )
#define NON_GHOST_COUNT(O) ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->non_ghost_count)) )
#define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;} #define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
#define UNLESS(E) if(!(E)) #define UNLESS(E) if(!(E))
#define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V) #define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
...@@ -112,21 +122,82 @@ if(self->state < 0 && self->jar) \ ...@@ -112,21 +122,82 @@ if(self->state < 0 && self->jar) \
{ \ { \
PyObject *r; \ PyObject *r; \
\ \
int *count = NON_GHOST_COUNT(self); \
if(count) \
{ \
(*count)++; \
self->ring.next = HOME(self); \
self->ring.prev = HOME(self)->prev; \
HOME(self)->prev->next = &self->ring; \
HOME(self)->prev = &self->ring; \
Py_INCREF(self); \
} \
self->state=cPersistent_CHANGED_STATE; \ self->state=cPersistent_CHANGED_STATE; \
UNLESS(r=callmethod1(self->jar,py_setstate,(PyObject*)self)) \ UNLESS(r=callmethod1(self->jar,py_setstate,(PyObject*)self)) \
{ \ { \
self->state=cPersistent_GHOST_STATE; \ ghostify(self); \
return ER; \ return ER; \
} \ } \
self->state=cPersistent_UPTODATE_STATE; \ self->state=cPersistent_UPTODATE_STATE; \
Py_DECREF(r); \ Py_DECREF(r); \
} }
#define KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self) \
if(HOME(self) && self->state>=0) { \
self->ring.prev->next = self->ring.next; \
self->ring.next->prev = self->ring.prev; \
self->ring.next = HOME(self); \
self->ring.prev = HOME(self)->prev; \
HOME(self)->prev->next = &self->ring; \
HOME(self)->prev = &self->ring; }
/****************************************************************************/ /****************************************************************************/
staticforward PyExtensionClass Pertype; staticforward PyExtensionClass Pertype;
static void
accessed(cPersistentObject *self)
{
KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
}
static void
ghostify(cPersistentObject *self)
{
int *count;
count = NON_GHOST_COUNT(self);
if(count && (self->state>=0))
{
(*count)--;
self->ring.next->prev = self->ring.prev;
self->ring.prev->next = self->ring.next;
self->ring.prev = NULL;
self->ring.next = NULL;
self->state = cPersistent_GHOST_STATE;
Py_DECREF(self);
}
else
{
self->state = cPersistent_GHOST_STATE;
}
}
static void
deallocated(cPersistentObject *self)
{
if(self->state>=0) ghostify(self);
if(self->cache)
{
PyObject *v=PyObject_CallMethod(self->cache,"_oid_unreferenced","O",self->oid);
if(!v) PyErr_Clear(); /* and explode later */
Py_XDECREF(v);
}
Py_XDECREF(self->jar);
Py_XDECREF(self->oid);
}
static int static int
changed(cPersistentObject *self) changed(cPersistentObject *self)
{ {
...@@ -185,7 +256,7 @@ Per___changed__(cPersistentObject *self, PyObject *args) ...@@ -185,7 +256,7 @@ Per___changed__(cPersistentObject *self, PyObject *args)
static PyObject * static PyObject *
Per__p_deactivate(cPersistentObject *self, PyObject *args) Per__p_deactivate(cPersistentObject *self, PyObject *args)
{ {
PyObject *dict; PyObject *dict,*dict2=NULL;
#ifdef DEBUG_LOG #ifdef DEBUG_LOG
if (idebug_log < 0) call_debug("reinit",self); if (idebug_log < 0) call_debug("reinit",self);
...@@ -197,13 +268,22 @@ Per__p_deactivate(cPersistentObject *self, PyObject *args) ...@@ -197,13 +268,22 @@ Per__p_deactivate(cPersistentObject *self, PyObject *args)
if (self->state==cPersistent_UPTODATE_STATE && self->jar && if (self->state==cPersistent_UPTODATE_STATE && self->jar &&
HasInstDict(self) && (dict=INSTANCE_DICT(self))) HasInstDict(self) && (dict=INSTANCE_DICT(self)))
{ {
dict2 = PyDict_Copy(dict);
PyDict_Clear(dict); PyDict_Clear(dict);
/* Note that we need to set to ghost state unless we are /* Note that we need to set to ghost state unless we are
called directly. Methods that override this need to called directly. Methods that override this need to
do the same! */ do the same! */
self->state=cPersistent_GHOST_STATE; ghostify(self);
} }
/* need to delay releasing the last reference on instance attributes
until after we have finished accounting for losing our state */
if(dict2)
{
PyDict_Clear(dict2);
Py_DECREF(dict2);
}
Py_INCREF(Py_None); Py_INCREF(Py_None);
return Py_None; return Py_None;
} }
...@@ -333,8 +413,8 @@ Per_dealloc(cPersistentObject *self) ...@@ -333,8 +413,8 @@ Per_dealloc(cPersistentObject *self)
#ifdef DEBUG_LOG #ifdef DEBUG_LOG
if(idebug_log < 0) call_debug("del",self); if(idebug_log < 0) call_debug("del",self);
#endif #endif
Py_XDECREF(self->jar); deallocated(self);
Py_XDECREF(self->oid); Py_XDECREF(self->cache);
Py_DECREF(self->ob_type); Py_DECREF(self->ob_type);
PyObject_DEL(self); PyObject_DEL(self);
} }
...@@ -387,7 +467,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name, ...@@ -387,7 +467,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
{ {
UPDATE_STATE_IF_NECESSARY(self, NULL); UPDATE_STATE_IF_NECESSARY(self, NULL);
self->atime=((long)(time(NULL)/3))%65536; KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
if (self->serial[7]=='\0' && self->serial[6]=='\0' && if (self->serial[7]=='\0' && self->serial[6]=='\0' &&
self->serial[5]=='\0' && self->serial[4]=='\0' && self->serial[5]=='\0' && self->serial[4]=='\0' &&
...@@ -419,7 +499,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name, ...@@ -419,7 +499,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
{ {
UPDATE_STATE_IF_NECESSARY(self, NULL); UPDATE_STATE_IF_NECESSARY(self, NULL);
self->atime=((long)(time(NULL)/3))%65536; KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
} }
return getattrf((PyObject *)self, oname); return getattrf((PyObject *)self, oname);
...@@ -466,6 +546,21 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v, ...@@ -466,6 +546,21 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
{ {
if(name[3]=='o' && name[4]=='i' && name[5]=='d' && ! name[6]) if(name[3]=='o' && name[4]=='i' && name[5]=='d' && ! name[6])
{ {
if(HOME(self))
{
int result;
if(!v)
{
PyErr_SetString(PyExc_ValueError,"can not delete the oid of a cached object");
return -1;
}
if(PyObject_Cmp(self->oid,v,&result)<0) return -1;
if(result)
{
PyErr_SetString(PyExc_ValueError,"can not change the oid of a cached object");
return -1;
}
}
Py_XINCREF(v); Py_XINCREF(v);
ASSIGN(self->oid, v); ASSIGN(self->oid, v);
return 0; return 0;
...@@ -509,7 +604,6 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v, ...@@ -509,7 +604,6 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
v=PyObject_GetAttr(OBJECT(self), py__p_deactivate); v=PyObject_GetAttr(OBJECT(self), py__p_deactivate);
if (v) { ASSIGN(v, PyObject_CallObject(v, NULL)); } if (v) { ASSIGN(v, PyObject_CallObject(v, NULL)); }
if (v) { Py_DECREF(v); } if (v) { Py_DECREF(v); }
self->state=cPersistent_GHOST_STATE;
return 0; return 0;
} }
if (PyObject_IsTrue(v)) return changed(self); if (PyObject_IsTrue(v)) return changed(self);
...@@ -521,8 +615,7 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v, ...@@ -521,8 +615,7 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
{ {
UPDATE_STATE_IF_NECESSARY(self, -1); UPDATE_STATE_IF_NECESSARY(self, -1);
/* Record access times */ KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
self->atime=((long)(time(NULL)/3))%65536;
if((! (*name=='_' && name[1]=='v' && name[2]=='_')) if((! (*name=='_' && name[1]=='v' && name[2]=='_'))
&& (self->state != cPersistent_CHANGED_STATE && self->jar) && (self->state != cPersistent_CHANGED_STATE && self->jar)
...@@ -680,9 +773,11 @@ truecPersistenceCAPI = { ...@@ -680,9 +773,11 @@ truecPersistenceCAPI = {
(getattrofunc)Per_getattro, /*tp_getattr with object key*/ (getattrofunc)Per_getattro, /*tp_getattr with object key*/
(setattrofunc)Per_setattro, /*tp_setattr with object key*/ (setattrofunc)Per_setattro, /*tp_setattr with object key*/
changed, changed,
accessed,
ghostify,
deallocated,
(intfunctionwithpythonarg)Per_setstate, (intfunctionwithpythonarg)Per_setstate,
(pergetattr)Per_getattr, (pergetattr)Per_getattr,
(persetattr)_setattro,
}; };
void void
......
src/Persistence/cPersistence.h
View file @ 226ee421
...@@ -18,12 +18,21 @@ ...@@ -18,12 +18,21 @@
#include "ExtensionClass.h" #include "ExtensionClass.h"
#include <time.h> #include <time.h>
#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid; char serial[8]; unsigned short atime; signed char state; unsigned char reserved;
#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid, *cache; CPersistentRing ring; char serial[8]; signed char state; unsigned char reserved[3];
#define cPersistent_GHOST_STATE -1 #define cPersistent_GHOST_STATE -1
#define cPersistent_UPTODATE_STATE 0 #define cPersistent_UPTODATE_STATE 0
#define cPersistent_CHANGED_STATE 1 #define cPersistent_CHANGED_STATE 1
#define cPersistent_STICKY_STATE 2 #define cPersistent_STICKY_STATE 2
struct ccobject_head_struct;
typedef struct CPersistentRing_struct
{
struct CPersistentRing_struct *prev;
struct CPersistentRing_struct *next;
} CPersistentRing;
typedef struct { typedef struct {
cPersistent_HEAD cPersistent_HEAD
} cPersistentObject; } cPersistentObject;
...@@ -36,6 +45,9 @@ typedef struct { ...@@ -36,6 +45,9 @@ typedef struct {
getattrofunc getattro; getattrofunc getattro;
setattrofunc setattro; setattrofunc setattro;
int (*changed)(cPersistentObject*); int (*changed)(cPersistentObject*);
void (*accessed)(cPersistentObject*);
void (*ghostify)(cPersistentObject*);
void (*deallocated)(cPersistentObject*);
int (*setstate)(PyObject*); int (*setstate)(PyObject*);
pergetattr pergetattro; pergetattr pergetattro;
persetattr persetattro; persetattr persetattro;
...@@ -59,11 +71,13 @@ static cPersistenceCAPIstruct *cPersistenceCAPI; ...@@ -59,11 +71,13 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
#define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O))) #define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O)))
#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE)) #define PER_GHOSTIFY(O) (cPersistenceCAPI->ghostify((cPersistentObject*)(O)))
#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
#define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE)) #define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
#define PER_DEL(O) Py_XDECREF((O)->jar); Py_XDECREF((O)->oid); #define PER_DEL(O) (cPersistenceCAPI->deallocated((cPersistentObject*)(O)))
#define PER_USE(O) \ #define PER_USE(O) \
(((O)->state != cPersistent_GHOST_STATE \ (((O)->state != cPersistent_GHOST_STATE \
...@@ -71,7 +85,7 @@ static cPersistenceCAPIstruct *cPersistenceCAPI; ...@@ -71,7 +85,7 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
? (((O)->state==cPersistent_UPTODATE_STATE) \ ? (((O)->state==cPersistent_UPTODATE_STATE) \
? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0) ? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0)
#define PER_ACCESSED(O) ((O)->atime=((long)(time(NULL)/3))%65536) #define PER_ACCESSED(O) (cPersistenceCAPI->accessed((cPersistentObject*)(O)))
#endif #endif
......
src/Persistence/cPickleCache.c
View file @ 226ee421
...@@ -2,448 +2,531 @@ ...@@ -2,448 +2,531 @@
Copyright (c) 2001, 2002 Zope Corporation and Contributors. Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved. All Rights Reserved.
This software is subject to the provisions of the Zope Public License, This software is subject to the provisions of the Zope Public License,
Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE FOR A PARTICULAR PURPOSE
****************************************************************************/ ****************************************************************************/
static char cPickleCache_doc_string[] =
static char cPickleCache_doc_string[] =
"Defines the PickleCache used by ZODB Connection objects.\n" "Defines the PickleCache used by ZODB Connection objects.\n"
"\n" "\n"
"$Id: cPickleCache.c,v 1.41 2002/03/08 18:36:14 jeremy Exp $\n"; "$Id: cPickleCache.c,v 1.42 2002/03/27 10:14:04 htrd Exp $\n";
/* Compute the current time in the units and range used for peristent #define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
objects. */ #define UNLESS(E) if(!(E))
#define PER_TIME() ((long)(time(NULL) / 3)) % 65536 #define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
#define OBJECT(O) ((PyObject*)O)
#define DONT_USE_CPERSISTENCECAPI #define DONT_USE_CPERSISTENCECAPI
#include "cPersistence.h" #include "cPersistence.h"
#include <time.h> #include <time.h>
#include <stddef.h>
#undef Py_FindMethod #undef Py_FindMethod
static PyObject *py_reload, *py__p_jar, *py__p_changed;
typedef struct { static PyObject *py__p_oid, *py_reload, *py__p_jar, *py__p_changed;
PyObject_HEAD
PyObject *data;
PyObject *jar;
PyObject *setklassstate;
int position;
int cache_size;
int cache_age;
/* Cache statistics */
int sum_deal;
int sum_deac;
double sum_age;
int n, na;
time_t last_check; /* Time of last gc */
double mean_age;
double mean_deal;
double mean_deac;
double df, dfa; /* Degees of freedom for above stats */
} ccobject;
#define WEIGHTING_PERIOD 600 /* define this for extra debugging checks, and lousy performance */
#define MUCH_RING_CHECKING 1
/* /* Do we want 'engine noise'.... abstract debugging output useful for
How to compute weighted means? visualizing cache behavior */
#if 0
#define ENGINE_NOISE(A) printf(A)
#else
#define ENGINE_NOISE(A) ((void)A)
#endif
Assume we have two means, a current mean, M, and a mean as of some /* the layout of this struct is the same as the start of ccobject_head in cPersistence.c */
time d seconds in the past, Md. The means have effective degrees typedef struct {
of freedom, N, and Nd. Where Nd is adjusted by d is some fashion. PyObject_HEAD
The combined mean is (M*N+Md*Nd)/(N+Nd). The degrees of freedom CPersistentRing ring_home;
of the combined mean, Nc, is N+Nd. Nd is computed by weighting int non_ghost_count;
an old degree of freedom with the weight: I/(I+d), where I is some int klass_count;
suitably chosen constant, which we will call a "weighting period". PyObject *data;
PyObject *jar;
*/ PyObject *setklassstate;
int cache_size;
int ring_lock;
int cache_drain_resistance;
} ccobject;
staticforward PyTypeObject Cctype; staticforward PyTypeObject Cctype;
staticforward int present_in_ring(ccobject *self,CPersistentRing *target);
staticforward int check_ring(ccobject *self,const char *context);
staticforward int cc_ass_sub(ccobject *self, PyObject *key, PyObject *v);
/* ---------------------------------------------------------------- */ /* ---------------------------------------------------------------- */
static int static PyObject *object_from_oid(ccobject *self,PyObject *key)
gc_item(ccobject *self, PyObject *key, PyObject *v, long now, int dt) /* somewhat of a replacement for PyDict_GetItem(self->data....
however this returns a *new* reference */
{ {
if (!(v && key)) PyObject *v = PyDict_GetItem(self->data, key);
return 0; if(!v) return NULL;
self->n++;
/* If there is at most one reference to this object, then the
cache has the only reference. It can be removed. */
if (v->ob_refcnt <= 1) {
self->sum_deal++;
/* XXX The fact that this works will iterating over
self->data with PyDict_Next() is an accident of the
current Python dictionary implementation. */
return PyDict_DelItem(self->data, key);
}
if (dt >= 0 && Py_INCREF(v);
(!PyExtensionClass_Check(v)) &&
((cPersistentObject*)v)->jar == self->jar /* I'm paranoid */ && return v;
((cPersistentObject*)v)->state == cPersistent_UPTODATE_STATE) {
now -= ((cPersistentObject*)v)->atime;
if (now < 0)
now += 65536;
self->na++;
self->sum_age += now;
if (now > dt) {
/* We have a cPersistent object that hasn't been used in
a while. Reinitialize it, hopefully freeing it's
state.
*/
self->sum_deac++;
if (PyObject_SetAttr(v, py__p_changed, Py_None) < 0)
PyErr_Clear();
}
}
return 0;
} }
static void static cPersistentObject *object_from_ring(ccobject *self,CPersistentRing *here,const char *context)
update_stats(ccobject *self, time_t now)
{ {
double d, deal, deac; /* Given a position in the LRU ring, return a borrowed
reference to the object at that point in the ring. The caller is
d = now - self->last_check; responsible for ensuring that this ring position really does
if(d < 1) correspond to a persistent object, although the debugging
return; version will double-check this. */
self->df *= WEIGHTING_PERIOD / (WEIGHTING_PERIOD + d); PyObject *object = (PyObject *)(((char *)here)-offsetof(cPersistentObject,ring));
self->dfa *= WEIGHTING_PERIOD / (WEIGHTING_PERIOD + d);
#ifdef MUCH_RING_CHECKING
self->mean_age = ((self->mean_age * self->dfa + self->sum_age)/ if(!PyExtensionInstance_Check(object))
(self->dfa + self->na)) * 3; {
self->sum_age = 0; PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered non-ExtensionClass object in %s",context);
return NULL;
deac = self->sum_deac / d; }
self->sum_deac = 0; if(!(((PyExtensionClass*)(object->ob_type))->class_flags & PERSISTENT_TYPE_FLAG))
self->mean_deac = ((self->mean_deac * self->dfa+deac)/ {
(self->dfa + self->na)); PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered non-persistent object in %s",context);
self->sum_deac = 0; return NULL;
}
self->dfa += self->na; if(((cPersistentObject*)object)->jar!=self->jar)
self->na = 0; {
PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered object from a different jar in %s",context);
deal=self->sum_deal/d; return NULL;
self->sum_deal = 0; }
self->mean_deal = ((self->mean_deal * self->df + deal)/ if(((cPersistentObject*)object)->cache!=(PyObject *)self)
(self->df +self->n)); {
self->sum_deal = 0; PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered broken ring in %s",context);
return NULL;
self->df += self->n; }
self->n = 0; #endif
return (cPersistentObject *)object;
self->last_check = now;
} }
static int static int
check_size(ccobject *self) scan_gc_items(ccobject *self,int target)
{ {
if (self->cache_size < 1) cPersistentObject *object;
return 0; int error;
return PyDict_Size(self->data); CPersistentRing placeholder;
CPersistentRing *here = self->ring_home.next;
#ifdef MUCH_RING_CHECKING
int safety_counter = self->cache_size*10;
if(safety_counter<10000) safety_counter = 10000;
#endif
while(1)
{
if(check_ring(self,"mid-gc")) return -1;
#ifdef MUCH_RING_CHECKING
if(!safety_counter--)
{
/* This loop has been running for a very long time.
It is possible that someone loaded a very large number of objects,
and now wants us to blow them all away. However it may
also indicate a logic error. If the loop has been running this
long then you really have to doubt it will ever terminate.
In the MUCH_RING_CHECKING build we prefer to raise an exception
here */
PyErr_SetString(PyExc_RuntimeError,"scan_gc_items safety counter exceeded");
return -1;
}
if(!present_in_ring(self,here))
{
/* Our current working position is no longer in the ring. Thats bad. */
PyErr_SetString(PyExc_RuntimeError,"working position fell out the ring, in scan_gc_items");
return -1;
}
#endif
if(here==&self->ring_home)
{
/* back to the home position. stop looking */
return 0;
}
/* At this point we know that the ring only contains nodes from
persistent objects, plus our own home node. We can safely
assume this is a persistent object now we know it is not the home */
object = object_from_ring(self,here,"scan_gc_items");
if(!object) return -1;
if(self->non_ghost_count<=target)
{
/* we are small enough */
return 0;
}
else if(object->state==cPersistent_UPTODATE_STATE)
{
/* deactivate it. This is the main memory saver. */
ENGINE_NOISE("G");
/* add a placeholder */
placeholder.next = here->next;
placeholder.prev = here;
here->next->prev = &placeholder;
here->next = &placeholder;
error = PyObject_SetAttr((PyObject *)object,py__p_changed,Py_None);
/* unlink the placeholder */
placeholder.next->prev=placeholder.prev;
placeholder.prev->next=placeholder.next;
here = placeholder.next;
if(error)
return -1; /* problem */
}
else
{
ENGINE_NOISE(".");
here = here->next;
}
}
} }
static int
gc_all_items(ccobject *self, int now, int dt) static PyObject *
lockgc(ccobject *self,int target_size)
{ {
PyObject *key, *v; if(self->ring_lock)
int i; {
Py_INCREF(Py_None);
return Py_None;
}
for(i = 0; PyDict_Next(self->data, &i, &key, &v); ) if(check_ring(self,"pre-gc")) return NULL;
if (gc_item(self, key, v, now, dt) < 0) ENGINE_NOISE("<");
return -1; self->ring_lock = 1;
return 0; if(scan_gc_items(self,target_size))
{
self->ring_lock = 0;
return NULL;
}
self->ring_lock = 0;
ENGINE_NOISE(">\n");
if(check_ring(self,"post-gc")) return NULL;
Py_INCREF(Py_None);
return Py_None;
} }
static int static PyObject *
fullgc(ccobject *self, int dt) cc_incrgc(ccobject *self, PyObject *args)
{ {
long now; int n=1;
if (check_size(self) <= 0) int starting_size = self->non_ghost_count;
return 0;
now = PER_TIME(); int target_size = self->cache_size;
dt /= 3;
if (gc_all_items(self, now, dt) < 0) if(self->cache_drain_resistance>=1)
return -1; {
self->position = 0; /* This cache will gradually drain down to a small size. Check
a (small) number of objects proportional to the current size */
if (now - self->last_check > 1) int target_size_2 = starting_size - 1 - starting_size/self->cache_drain_resistance;
update_stats(self, now); if(target_size_2<target_size)
target_size = target_size_2;
return 0;
}
static int
reallyfullgc(ccobject *self, int dt)
{
int l, last;
time_t now;
last = check_size(self);
if (last <= 0)
return 0;
now = PER_TIME();
/* Units are 3 seconds */
dt /= 3;
/* First time through should get refcounts to 1 */
if (gc_all_items(self, now, dt) < 0)
return -1;
l = PyDict_Size(self->data);
if (l < 0)
return -1;
/* Now continue to collect until the size of the cache stops
decreasing. */
while (l < last) {
if (gc_all_items(self, now, dt) < 0)
return -1;
last = l;
l = PyDict_Size(self->data);
if (l < 0)
return -1;
} }
if (now - self->last_check > 1)
update_stats(self, now);
self->position = 0;
return 0;
}
static int UNLESS (PyArg_ParseTuple(args, "|i",&n)) return NULL;
maybegc(ccobject *self, PyObject *thisv)
{
int n, s, size, dt;
long now;
PyObject *key=0, *v=0;
s = check_size(self);
if (s <= 0)
return 0;
now = PER_TIME();
size = self->cache_size;
self->cache_size = 0;
/* Decide how many objects to look at */
n = (s - size) / 10;
if (n < 3)
n = 3;
/* Decide how much time to give them before deactivating them */
s = 8 * size / s;
if (s > 100)
s = 100;
dt = (long)(self->cache_age * (0.2 + 0.1 * s));
/* Units are 3 seconds */
dt /= 3;
while (--n >= 0) {
if (PyDict_Next(self->data, &(self->position), &key, &v)) {
if (v != thisv && gc_item(self, key, v, now, dt) < 0) {
self->cache_size=size;
return -1;
}
}
else
self->position = 0;
}
self->cache_size = size;
if (now - self->last_check > 1)
update_stats(self, now);
return 0; return lockgc(self,target_size);
} }
static PyObject * static PyObject *
cc_full_sweep(ccobject *self, PyObject *args) cc_full_sweep(ccobject *self, PyObject *args)
{ {
int dt = self->cache_age; int dt=0;
if (!PyArg_ParseTuple(args, "|i:full_sweep", &dt)) UNLESS(PyArg_ParseTuple(args, "|i", &dt)) return NULL;
return NULL; return lockgc(self,0);
if (dt < -1) {
PyErr_SetString(PyExc_ValueError, "age must be >= -1");
return NULL;
}
if (fullgc(self, dt) == -1)
return NULL;
Py_INCREF(Py_None);
return Py_None;
} }
static PyObject * static PyObject *
cc_reallyfull_sweep(ccobject *self, PyObject *args) cc_reallyfull_sweep(ccobject *self, PyObject *args)
{ {
int dt = self->cache_age; int dt=0;
if (!PyArg_ParseTuple(args, "|i:minimize", &dt)) UNLESS(PyArg_ParseTuple(args, "|i", &dt)) return NULL;
return NULL; return lockgc(self,0);
if (dt < -1) {
PyErr_SetString(PyExc_ValueError, "age must be >= -1");
return NULL;
}
if (reallyfullgc(self, dt) == -1)
return NULL;
Py_INCREF(Py_None);
return Py_None;
} }
static PyObject * static void
cc_incrgc(ccobject *self, PyObject *args) _invalidate(ccobject *self, PyObject *key)
{ {
int n = 1; PyObject *v=object_from_oid(self, key);
if (!PyArg_ParseTuple(args, "|i:incrgr", &n)) if(!v)
return NULL; {
/* shouldnt this be an error? for now Ill follow Jims lead */
PyErr_Clear();
}
else
{
if (PyExtensionClass_Check(v))
{
if(v->ob_refcnt <= 1)
{
self->klass_count--;
if (PyDict_DelItem(self->data, key) < 0)
PyErr_Clear();
}
else
{
v=PyObject_CallFunction(self->setklassstate,
"O", v);
if (v) Py_DECREF(v);
else PyErr_Clear();
}
}
else
{
if(PyObject_DelAttr(v,py__p_changed) < 0)
PyErr_Clear();
}
Py_DECREF(v);
}
}
for (; --n >= 0;) static PyObject *
if (maybegc(self, NULL) < 0) cc_invalidate(ccobject *self, PyObject *args)
return NULL; {
PyObject *inv, *key, *v;
int i;
if (PyArg_ParseTuple(args, "O!", &PyDict_Type, &inv)) {
for (i=0; PyDict_Next(inv, &i, &key, &v); )
if (key==Py_None)
{ /* Eek some nitwit invalidated everything! */
for (i=0; PyDict_Next(self->data, &i, &key, &v); )
_invalidate(self, key);
break;
}
else
_invalidate(self, key);
PyDict_Clear(inv);
}
else {
PyErr_Clear();
UNLESS (PyArg_ParseTuple(args, "O", &inv)) return NULL;
if (PyString_Check(inv))
_invalidate(self, inv);
else if (inv==Py_None) /* All */
for (i=0; PyDict_Next(self->data, &i, &key, &v); )
_invalidate(self, key);
else {
int l;
PyErr_Clear();
if ((l=PyObject_Length(inv)) < 0) return NULL;
for(i=l; --i >= 0; )
{
UNLESS (key=PySequence_GetItem(inv, i)) return NULL;
_invalidate(self, key);
Py_DECREF(key);
}
PySequence_DelSlice(inv, 0, l);
}
}
Py_INCREF(Py_None); Py_INCREF(Py_None);
return Py_None; return Py_None;
} }
static void
_invalidate(ccobject *self, PyObject *key) static PyObject *
cc_get(ccobject *self, PyObject *args)
{ {
PyObject *v = PyDict_GetItem(self->data, key); PyObject *r, *key, *d=0;
if (!v) UNLESS (PyArg_ParseTuple(args,"O|O", &key, &d)) return NULL;
return;
if (PyExtensionClass_Check(v)) UNLESS (r=(PyObject *)object_from_oid(self, key))
if (v->ob_refcnt <= 1) { {
self->sum_deal++; if (d)
if (PyDict_DelItem(self->data, key) < 0) {
PyErr_Clear(); PyErr_Clear();
} else { r=d;
PyObject *t = PyTuple_New(1); Py_INCREF(r);
if (t) { }
PyTuple_SET_ITEM(t, 0, v); else
v = PyObject_CallObject(self->setklassstate, t); {
/* Set tuple element to NULL so that deallocating the PyErr_SetObject(PyExc_KeyError, key);
tuple does not decref t. return NULL;
*/
PyTuple_SET_ITEM(t, 0, NULL);
Py_DECREF(t);
} else
v = t;
if (v)
Py_DECREF(v);
else
PyErr_Clear();
} }
else if (PyObject_DelAttr(v, py__p_changed) < 0) }
PyErr_Clear();
return r;
} }
static void static PyObject *
_invalidate_all(ccobject *self) cc_klass_items(ccobject *self, PyObject *args)
{ {
PyObject *key, *v; PyObject *l,*k,*v;
int i; int p = 0;
if(!PyArg_ParseTuple(args,"")) return NULL;
l = PyList_New(0);
if(!l) return NULL;
while(PyDict_Next(self->data, &p, &k, &v))
{
if(PyExtensionClass_Check(v))
{
v=PyObject_CallMethod(l,"append","((OO))",k,v);
if(!v)
{
Py_DECREF(l);
return NULL;
}
}
}
for (i = 0; PyDict_Next(self->data, &i, &key, &v); ) return l;
_invalidate(self, key);
} }
static PyObject * static PyObject *
cc_invalidate(ccobject *self, PyObject *args) cc_lru_items(ccobject *self, PyObject *args)
{ {
PyObject *inv, *key, *v; PyObject *l;
int i; CPersistentRing *here;
if (!PyArg_ParseTuple(args, "O:invalidate", &inv)) if(!PyArg_ParseTuple(args,"")) return NULL;
return NULL;
if (PyDict_Check(inv)) { if(self->ring_lock)
for (i = 0; PyDict_Next(inv, &i, &key, &v); ) {
if (key == Py_None) { PyErr_SetString(PyExc_ValueError,".lru_items() is unavailable during garbage collection");
/* Eek some nitwit invalidated everything! */ return NULL;
_invalidate_all(self);
break;
}
else
_invalidate(self, key);
PyDict_Clear(inv);
} else if (PyString_Check(inv))
_invalidate(self, inv);
else if (inv == Py_None) /* All */
_invalidate_all(self);
else {
int l = PyObject_Length(inv);
if (l < 0)
return NULL;
for (i = l; --i >= 0; ) {
key = PySequence_GetItem(inv, i);
if (!key)
return NULL;
_invalidate(self, key);
Py_DECREF(key);
}
PySequence_DelSlice(inv, 0, l);
} }
Py_INCREF(Py_None); if(check_ring(self,"pre-cc_items")) return NULL;
return Py_None;
l = PyList_New(0);
if(!l) return NULL;
here = self->ring_home.next;
while(here!=&self->ring_home)
{
cPersistentObject *object = object_from_ring(self,here,"cc_items");
PyObject *v;
if(!object)
{
Py_DECREF(l);
return NULL;
}
v=PyObject_CallMethod(l,"append","((OO))",object->oid,object);
if(!v)
{
Py_DECREF(l);
return NULL;
}
Py_DECREF(v);
here = here->next;
}
return l;
} }
static PyObject * static PyObject *
cc_get(ccobject *self, PyObject *args) cc_oid_unreferenced(ccobject *self, PyObject *args)
{ {
PyObject *r, *key, *d = NULL; PyObject *oid,*v;
if(!PyArg_ParseTuple(args,"O",&oid)) return NULL;
if (!PyArg_ParseTuple(args, "O|O:get", &key, &d)) v = PyDict_GetItem(self->data, oid);
return NULL; if(!v) return NULL;
r = PyDict_GetItem(self->data, key); if(v->ob_refcnt)
if (!r) { {
if (d) PyErr_Format(PyExc_ValueError,"object has reference count of %d, should be zero",v->ob_refcnt);
r = d; return NULL;
else {
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
}
} }
Py_INCREF(r); /* Need to be very hairy here because a dictionary is about
return r; to decref an already deleted object */
#ifdef Py_TRACE_REFS
#error "this code path has not been tested - Toby Dickenson"
_Py_NewReference(v);
/* it may be a problem that v->ob_type is still NULL? */
#else
Py_INCREF(v);
#endif
if(v->ob_refcnt!=1)
{
PyErr_SetString(PyExc_ValueError,"refcount is not 1 after resurrection");
return NULL;
}
/* return the stolen reference */
Py_INCREF(v);
PyDict_DelItem(self->data, oid);
if(v->ob_refcnt!=1)
{
PyErr_SetString(PyExc_ValueError,"refcount is not 1 after removal from dict");
return NULL;
}
/* undo the temporary resurrection */
#ifdef Py_TRACE_REFS
_Py_ForgetReference(v);
#else
v->ob_refcnt=0;
#endif
Py_INCREF(Py_None);
return Py_None;
} }
static struct PyMethodDef cc_methods[] = { static struct PyMethodDef cc_methods[] = {
{"_oid_unreferenced", (PyCFunction)cc_oid_unreferenced, METH_VARARGS,
NULL
},
{"lru_items", (PyCFunction)cc_lru_items, METH_VARARGS,
"List (oid, object) pairs from the lru list, as 2-tuples.\n"
},
{"klass_items", (PyCFunction)cc_klass_items, METH_VARARGS,
"List (oid, object) pairs of cached persistent classes.\n"
},
{"full_sweep", (PyCFunction)cc_full_sweep, METH_VARARGS, {"full_sweep", (PyCFunction)cc_full_sweep, METH_VARARGS,
"full_sweep([age]) -- Perform a full sweep of the cache\n\n" "full_sweep([age]) -- Perform a full sweep of the cache\n\n"
"Make a single pass through the cache, removing any objects that are no\n" "Make a single pass through the cache, removing any objects that are no\n"
"longer referenced, and deactivating objects that have not been\n" "longer referenced, and deactivating enough objects to bring\n"
"accessed in the number of seconds given by 'age'. " "the cache under its size limit\n"
"'age defaults to the cache age.\n" "The optional 'age' parameter is ignored.\n"
}, },
{"minimize", (PyCFunction)cc_reallyfull_sweep, METH_VARARGS, {"minimize", (PyCFunction)cc_reallyfull_sweep, METH_VARARGS,
"minimize([age]) -- Remove as many objects as possible\n\n" "minimize([age]) -- Remove as many objects as possible\n\n"
"Make multiple passes through the cache, removing any objects that are no\n" "Make multiple passes through the cache, removing any objects that are no\n"
"longer referenced, and deactivating objects that have not been\n" "longer referenced, and deactivating enough objects to bring the"
"accessed in the number of seconds given by 'age'. 'age defaults to 0.\n" " cache under its size limit\n"
"The option 'age' parameter is ignored.\n"
}, },
{"incrgc", (PyCFunction)cc_incrgc, METH_VARARGS, {"incrgc", (PyCFunction)cc_incrgc, METH_VARARGS,
"incrgc() -- Perform incremental garbage collection"}, "incrgc([n]) -- Perform incremental garbage collection\n\n"
"Some other implementations support an optional parameter 'n' which\n"
"indicates a repetition count; this value is ignored.\n"},
{"invalidate", (PyCFunction)cc_invalidate, METH_VARARGS, {"invalidate", (PyCFunction)cc_invalidate, METH_VARARGS,
"invalidate(oids) -- invalidate one, many, or all ids"}, "invalidate(oids) -- invalidate one, many, or all ids"},
{"get", (PyCFunction)cc_get, METH_VARARGS, {"get", (PyCFunction)cc_get, METH_VARARGS,
...@@ -454,115 +537,102 @@ static struct PyMethodDef cc_methods[] = { ...@@ -454,115 +537,102 @@ static struct PyMethodDef cc_methods[] = {
static ccobject * static ccobject *
newccobject(PyObject *jar, int cache_size, int cache_age) newccobject(PyObject *jar, int cache_size, int cache_age)
{ {
ccobject *self; ccobject *self;
self = PyObject_NEW(ccobject, &Cctype); UNLESS(self = PyObject_NEW(ccobject, &Cctype)) return NULL;
if (!self) self->setklassstate=self->jar=NULL;
if((self->data=PyDict_New()))
{
self->jar=jar;
Py_INCREF(jar);
UNLESS (self->setklassstate=PyObject_GetAttrString(jar, "setklassstate"))
return NULL; return NULL;
self->setklassstate = self->jar = NULL; self->cache_size=cache_size;
self->data = PyDict_New(); self->non_ghost_count=0;
if (self->data) { self->klass_count=0;
self->jar=jar; self->cache_drain_resistance=0;
Py_INCREF(jar); self->ring_lock=0;
self->setklassstate = PyObject_GetAttrString(jar, "setklassstate"); self->ring_home.next = &self->ring_home;
if (!self->setklassstate) { self->ring_home.prev = &self->ring_home;
Py_DECREF(jar); return self;
Py_DECREF(self->data);
goto error;
}
self->position = 0;
self->cache_size = cache_size;
self->cache_age = cache_age < 1 ? 1 : cache_age;
self->sum_deal = 0;
self->sum_deac = 0;
self->sum_age = 0;
self->mean_deal = 0;
self->mean_deac = 0;
self->mean_age = 0;
self->df = 1;
self->dfa = 1;
self->n = 0;
self->na = 0;
self->last_check = time(NULL);
return self;
} }
error: Py_DECREF(self);
Py_DECREF(self); return NULL;
return NULL;
} }
static void static void
cc_dealloc(ccobject *self) cc_dealloc(ccobject *self)
{ {
Py_XDECREF(self->data); Py_XDECREF(self->data);
Py_XDECREF(self->jar); Py_XDECREF(self->jar);
Py_XDECREF(self->setklassstate); Py_XDECREF(self->setklassstate);
PyObject_DEL(self); PyMem_DEL(self);
} }
static PyObject * static PyObject *
cc_getattr(ccobject *self, char *name) cc_getattr(ccobject *self, char *name)
{ {
PyObject *r; PyObject *r;
if (*name == 'c') { if(check_ring(self,"getattr")) return NULL;
if(strcmp(name, "cache_age") == 0)
return PyInt_FromLong(self->cache_age); if(*name=='c')
if(strcmp(name, "cache_size") == 0) {
return PyInt_FromLong(self->cache_size); if(strcmp(name,"cache_age")==0)
if(strcmp(name, "cache_mean_age") == 0) return PyInt_FromLong(0); /* this cache does not use this value */
return PyFloat_FromDouble(self->mean_age); if(strcmp(name,"cache_size")==0)
if(strcmp(name, "cache_mean_deal") == 0) return PyInt_FromLong(self->cache_size);
return PyFloat_FromDouble(self->mean_deal); if(strcmp(name,"cache_drain_resistance")==0)
if(strcmp(name, "cache_mean_deac") == 0) return PyInt_FromLong(self->cache_drain_resistance);
return PyFloat_FromDouble(self->mean_deac); if(strcmp(name,"cache_non_ghost_count")==0)
if(strcmp(name, "cache_df") == 0) return PyInt_FromLong(self->non_ghost_count);
return PyFloat_FromDouble(self->df); if(strcmp(name,"cache_klass_count")==0)
if(strcmp(name, "cache_dfa") == 0) return PyInt_FromLong(self->klass_count);
return PyFloat_FromDouble(self->dfa); if(strcmp(name,"cache_data")==0)
if(strcmp(name, "cache_last_gc_time") == 0) {
return PyFloat_FromDouble(self->last_check); /* now a copy of our data; the ring is too fragile */
if(strcmp(name, "cache_data") == 0) { return PyDict_Copy(self->data);
Py_INCREF(self->data);
return self->data;
} }
} }
if ((strcmp(name, "has_key") == 0) if((*name=='h' && strcmp(name, "has_key")==0) ||
|| (strcmp(name, "items") == 0) (*name=='i' && strcmp(name, "items")==0) ||
|| (strcmp(name, "keys") == 0)) (*name=='k' && strcmp(name, "keys")==0)
return PyObject_GetAttrString(self->data, name); )
return PyObject_GetAttrString(self->data, name);
r = Py_FindMethod(cc_methods, (PyObject *)self, name);
if (!r) { if((r=Py_FindMethod(cc_methods, (PyObject *)self, name)))
PyErr_Clear();
return PyObject_GetAttrString(self->data, name);
}
return r; return r;
PyErr_Clear();
return PyObject_GetAttrString(self->data, name);
} }
static int static int
cc_setattr(ccobject *self, char *name, PyObject *value) cc_setattr(ccobject *self, char *name, PyObject *value)
{ {
if (value) { if(value)
{
int v; int v;
if (strcmp(name, "cache_age") == 0) { if(strcmp(name,"cache_age")==0)
v = PyInt_AsLong(value); {
if (v == -1 && PyErr_Occurred()) /* this cache doesnt use the age */
return -1;
if (v > 0)
self->cache_age = v;
return 0; return 0;
} }
if (strcmp(name, "cache_size") == 0) { if(strcmp(name,"cache_size")==0)
v = PyInt_AsLong(value); {
if (v == -1 && PyErr_Occurred()) UNLESS(PyArg_Parse(value,"i",&v)) return -1;
return -1; self->cache_size=v;
self->cache_size = v;
return 0; return 0;
} }
}
if(strcmp(name,"cache_drain_resistance")==0)
{
UNLESS(PyArg_Parse(value,"i",&v)) return -1;
self->cache_drain_resistance=v;
return 0;
}
}
PyErr_SetString(PyExc_AttributeError, name); PyErr_SetString(PyExc_AttributeError, name);
return -1; return -1;
} }
...@@ -570,7 +640,7 @@ cc_setattr(ccobject *self, char *name, PyObject *value) ...@@ -570,7 +640,7 @@ cc_setattr(ccobject *self, char *name, PyObject *value)
static int static int
cc_length(ccobject *self) cc_length(ccobject *self)
{ {
return PyDict_Size(self->data); return PyObject_Length(self->data);
} }
static PyObject * static PyObject *
...@@ -578,39 +648,245 @@ cc_subscript(ccobject *self, PyObject *key) ...@@ -578,39 +648,245 @@ cc_subscript(ccobject *self, PyObject *key)
{ {
PyObject *r; PyObject *r;
r = PyDict_GetItem(self->data, key); if(check_ring(self,"__getitem__")) return NULL;
if (!r) {
PyErr_SetObject(PyExc_KeyError, key); UNLESS (r=(PyObject *)object_from_oid(self, key))
return NULL; {
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
} }
Py_INCREF(r);
return r; return r;
} }
static int static int
cc_ass_sub(ccobject *self, PyObject *key, PyObject *v) cc_ass_sub(ccobject *self, PyObject *key, PyObject *v)
{ {
if (v) { int result;
if (PyExtensionClass_Check(v) if(v)
|| {
(PyExtensionInstance_Check(v) if( ( PyExtensionInstance_Check(v) &&
&& (((PyExtensionClass*)(v->ob_type))->class_flags & PERSISTENT_TYPE_FLAG) &&
(((PyExtensionClass*)(v->ob_type))->class_flags (v->ob_type->tp_basicsize >= sizeof(cPersistentObject))
& PERSISTENT_TYPE_FLAG) )
&& ||
(v->ob_type->tp_basicsize >= sizeof(cPersistentObject)) PyExtensionClass_Check(v)
) )
) {
return PyDict_SetItem(self->data, key, v); PyObject *oid = PyObject_GetAttr(v,py__p_oid);
PyObject *object_again;
PyErr_SetString(PyExc_ValueError, if(!oid)
"Cache values must be persistent objects or classes."); {
return -1; return -1;
}
if(PyObject_Cmp(key,oid,&result))
{
Py_DECREF(oid);
return -1;
}
Py_DECREF(oid);
if(result)
{
PyErr_SetString(PyExc_ValueError,"key must be the same as the object's oid attribute");
return -1;
}
object_again = object_from_oid(self, key);
if(object_again)
{
if(object_again!=v)
{
Py_DECREF(object_again);
PyErr_SetString(PyExc_ValueError,"Can not re-register object under a different oid");
return -1;
}
else
{
/* re-register under the same oid - no work needed */
Py_DECREF(object_again);
return 0;
}
}
if(PyExtensionClass_Check(v))
{
if(PyDict_SetItem(self->data, key, v)) return -1;
self->klass_count++;
return 0;
}
else
{
if(((cPersistentObject*)v)->cache)
{
if(((cPersistentObject*)v)->cache==(PyObject *)self)
{
/* This object is already one of ours, which is ok.
It would be very strange if someone was trying to register the
same object under a different key */
}
else
{
/* This object is already in a different cache. */
PyErr_SetString(PyExc_ValueError, "Cache values may only be in one cache.");
return -1;
}
}
if(check_ring(self,"pre-setitem")) return -1;
if(PyDict_SetItem(self->data, key, v)) return -1;
Py_INCREF(self);
((cPersistentObject*)v)->cache = (PyObject *)self;
if(((cPersistentObject*)v)->state>=0)
{
/* insert this non-ghost object into the ring just behind the home position */
self->non_ghost_count++;
((cPersistentObject*)v)->ring.next = &self->ring_home;
((cPersistentObject*)v)->ring.prev = self->ring_home.prev;
self->ring_home.prev->next = &((cPersistentObject*)v)->ring;
self->ring_home.prev = &((cPersistentObject*)v)->ring;
}
else
{
/* steal a reference from the dictionary; ghosts have a weak reference */
Py_DECREF(v);
}
if(check_ring(self,"post-setitem")) return -1;
return 0;
}
}
else
{
PyErr_SetString(PyExc_ValueError, "Cache values must be persistent objects.");
return -1;
}
}
else
{
/* unlink this item from the ring */
if(check_ring(self,"pre-delitem")) return -1;
v = (PyObject *)object_from_oid(self,key);
if(!v) return -1;
if(PyExtensionClass_Check(v))
{
self->klass_count--;
}
else
{
if(((cPersistentObject*)v)->state>=0)
{
self->non_ghost_count--;
((cPersistentObject*)v)->ring.next->prev = ((cPersistentObject*)v)->ring.prev;
((cPersistentObject*)v)->ring.prev->next = ((cPersistentObject*)v)->ring.next;
((cPersistentObject*)v)->ring.prev = NULL;
((cPersistentObject*)v)->ring.next = NULL;
}
else
{
/* This is a ghost object, so we havent kept a reference count on it.
For it have stayed alive this long someone else must be keeping a reference
to it. Therefore we need to temporarily give it back a reference count
before calling DelItem below */
Py_INCREF(v);
}
Py_DECREF(((cPersistentObject*)v)->cache);
((cPersistentObject*)v)->cache = NULL;
}
Py_DECREF(v);
if(PyDict_DelItem(self->data, key))
{
PyErr_SetString(PyExc_RuntimeError,
"unexpectedly couldnt remove key in cc_ass_sub");
return -1;
}
if(check_ring(self,"post-delitem")) return -1;
return 0;
} }
return PyDict_DelItem(self->data, key);
} }
static int _check_ring(ccobject *self,const char *context)
{
CPersistentRing *here = &(self->ring_home);
int expected = 1+self->non_ghost_count;
int total = 0;
do
{
if(++total>(expected+10)) return 3; /* ring too big, by a large margin */
if(!here->next) return 4; /* various linking problems */
if(!here->prev) return 5;
if(!here->next->prev) return 7;
if(!here->prev->next) return 8;
if(here->prev->next!=here) return 9;
if(here->next->prev!=here) return 10;
if(!self->ring_lock)
{
/* if the ring must be locked then it only contains object other than persistent instances */
if(here!=&self->ring_home)
{
cPersistentObject *object = object_from_ring(self,here,context);
if(!object) return 12;
if(object->state==cPersistent_GHOST_STATE)
return 13;
}
}
here = here->next;
}
while(here!=&self->ring_home);
if(self->ring_lock)
{
if(total<expected) return 6; /* ring too small; too big is ok when locked */
}
else
{
if(total!=expected) return 14; /* ring size wrong, or bad ghost accounting */
}
return 0;
}
static int check_ring(ccobject *self,const char *context)
{
#ifdef MUCH_RING_CHECKING
int code=_check_ring(self,context);
if(code)
{
/*printf(stderr,"BROKEN RING (code %d) in %s, size %d\n",code,context,PyDict_Size(self->data));*/
PyErr_Format(PyExc_RuntimeError,"broken ring (code %d) in %s, size %d",code,context,PyDict_Size(self->data));
return code;
}
#endif
return 0;
}
static int
present_in_ring(ccobject *self,CPersistentRing *target)
{
CPersistentRing *here = self->ring_home.next;
while(1)
{
if(here==target)
{
return 1;
}
if(here==&self->ring_home)
{
/* back to the home position, and we didnt find it */
return 0;
}
here = here->next;
}
}
static PyMappingMethods cc_as_mapping = { static PyMappingMethods cc_as_mapping = {
(inquiry)cc_length, /*mp_length*/ (inquiry)cc_length, /*mp_length*/
(binaryfunc)cc_subscript, /*mp_subscript*/ (binaryfunc)cc_subscript, /*mp_subscript*/
...@@ -636,6 +912,10 @@ static PyTypeObject Cctype = { ...@@ -636,6 +912,10 @@ static PyTypeObject Cctype = {
(hashfunc)0, /*tp_hash*/ (hashfunc)0, /*tp_hash*/
(ternaryfunc)0, /*tp_call*/ (ternaryfunc)0, /*tp_call*/
(reprfunc)0, /*tp_str*/ (reprfunc)0, /*tp_str*/
/* Space for future expansion */
0L,0L,0L,0L,
""
}; };
static PyObject * static PyObject *
...@@ -644,9 +924,9 @@ cCM_new(PyObject *self, PyObject *args) ...@@ -644,9 +924,9 @@ cCM_new(PyObject *self, PyObject *args)
int cache_size=100, cache_age=1000; int cache_size=100, cache_age=1000;
PyObject *jar; PyObject *jar;
if (!PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age)) UNLESS(PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age))
return NULL; return NULL;
return (PyObject *)newccobject(jar, cache_size, cache_age); return (PyObject*)newccobject(jar, cache_size,cache_age);
} }
static struct PyMethodDef cCM_methods[] = { static struct PyMethodDef cCM_methods[] = {
...@@ -657,12 +937,11 @@ static struct PyMethodDef cCM_methods[] = { ...@@ -657,12 +937,11 @@ static struct PyMethodDef cCM_methods[] = {
void void
initcPickleCache(void) initcPickleCache(void)
{ {
PyObject *m; PyObject *m, *d;
Cctype.ob_type = &PyType_Type; Cctype.ob_type=&PyType_Type;
if (!ExtensionClassImported) UNLESS(ExtensionClassImported) return;
return;
m = Py_InitModule4("cPickleCache", cCM_methods, cPickleCache_doc_string, m = Py_InitModule4("cPickleCache", cCM_methods, cPickleCache_doc_string,
(PyObject*)NULL, PYTHON_API_VERSION); (PyObject*)NULL, PYTHON_API_VERSION);
...@@ -670,4 +949,15 @@ initcPickleCache(void) ...@@ -670,4 +949,15 @@ initcPickleCache(void)
py_reload = PyString_InternFromString("reload"); py_reload = PyString_InternFromString("reload");
py__p_jar = PyString_InternFromString("_p_jar"); py__p_jar = PyString_InternFromString("_p_jar");
py__p_changed = PyString_InternFromString("_p_changed"); py__p_changed = PyString_InternFromString("_p_changed");
py__p_oid = PyString_InternFromString("_p_oid");
d = PyModule_GetDict(m);
PyDict_SetItemString(d,"cache_variant",PyString_FromString("stiff/c"));
#ifdef MUCH_RING_CHECKING
PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(1));
#else
PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(0));
#endif
} }
src/ZODB/Connection.py
View file @ 226ee421
...@@ -13,14 +13,14 @@ ...@@ -13,14 +13,14 @@
############################################################################## ##############################################################################
"""Database connection support """Database connection support
$Id: Connection.py,v 1.63 2002/02/11 23:40:42 gvanrossum Exp $""" $Id: Connection.py,v 1.64 2002/03/27 10:14:03 htrd Exp $"""
__version__='$Revision: 1.63 $'[11:-2] __version__='$Revision: 1.64 $'[11:-2]
from cPickleCache import PickleCache from cPickleCache import PickleCache, MUCH_RING_CHECKING
from POSException import ConflictError, ReadConflictError from POSException import ConflictError, ReadConflictError
from ExtensionClass import Base from ExtensionClass import Base
import ExportImport, TmpStore import ExportImport, TmpStore
from zLOG import LOG, ERROR, BLATHER from zLOG import LOG, ERROR, BLATHER, WARNING
from coptimizations import new_persistent_id from coptimizations import new_persistent_id
from ConflictResolution import ResolvedSerial from ConflictResolution import ResolvedSerial
...@@ -32,6 +32,11 @@ from types import StringType, ClassType ...@@ -32,6 +32,11 @@ from types import StringType, ClassType
global_code_timestamp = 0 global_code_timestamp = 0
if MUCH_RING_CHECKING:
# To get rid of this warning, change the define inside cPickleCache.c and recompile.
LOG('ZODB',WARNING, 'Using cPickleCache with low performance (but extra debugging checks)')
del MUCH_RING_CHECKING
def updateCodeTimestamp(): def updateCodeTimestamp():
''' '''
Called after changes are made to persistence-based classes. Called after changes are made to persistence-based classes.
...@@ -65,12 +70,35 @@ class Connection(ExportImport.ExportImport): ...@@ -65,12 +70,35 @@ class Connection(ExportImport.ExportImport):
"""Create a new Connection""" """Create a new Connection"""
self._version=version self._version=version
self._cache=cache=PickleCache(self, cache_size, cache_deactivate_after) self._cache=cache=PickleCache(self, cache_size, cache_deactivate_after)
if version:
# Caches for versions end up empty if the version
# is not used for a while. Non-version caches
# keep their content indefinitely.
self._cache.cache_drain_resistance = 100
self._incrgc=self.cacheGC=cache.incrgc self._incrgc=self.cacheGC=cache.incrgc
self._invalidated=d={} self._invalidated=d={}
self._invalid=d.has_key self._invalid=d.has_key
self._committed=[] self._committed=[]
self._code_timestamp = global_code_timestamp self._code_timestamp = global_code_timestamp
def _cache_items(self):
# find all items on the lru list
items = self._cache.lru_items()
# fine everything. some on the lru list, some not
everything = self._cache.cache_data
# remove those items that are on the lru list
for k,v in items:
del everything[k]
# return a list of [ghosts....not recently used.....recently used]
return everything.items() + items
def __repr__(self):
if self._version:
ver = ' (in version %s)' % `self._version`
else:
ver = ''
return '<Connection at %08x%s>' % (id(self),ver)
def _breakcr(self): def _breakcr(self):
try: del self._cache try: del self._cache
except: pass except: pass
...@@ -414,9 +442,9 @@ class Connection(ExportImport.ExportImport): ...@@ -414,9 +442,9 @@ class Connection(ExportImport.ExportImport):
for oid in creating: for oid in creating:
o=cache_get(oid, None) o=cache_get(oid, None)
if o is not None: if o is not None:
del cache[oid]
del o._p_jar del o._p_jar
del o._p_oid del o._p_oid
del cache[oid]
#XXX #XXX
...@@ -441,9 +469,14 @@ class Connection(ExportImport.ExportImport): ...@@ -441,9 +469,14 @@ class Connection(ExportImport.ExportImport):
def root(self): return self['\0\0\0\0\0\0\0\0'] def root(self): return self['\0\0\0\0\0\0\0\0']
def setstate(self, object): def setstate(self, object):
oid=object._p_oid
if self._storage is None:
msg = "Shouldn't load state for %s when the connection is closed" % `oid`
LOG('ZODB',ERROR, msg)
raise RuntimeError(msg)
try: try:
oid=object._p_oid
p, serial = self._storage.load(oid, self._version) p, serial = self._storage.load(oid, self._version)
# XXX this is quite conservative! # XXX this is quite conservative!
......
src/ZODB/DB.py
View file @ 226ee421
...@@ -13,8 +13,8 @@ ...@@ -13,8 +13,8 @@
############################################################################## ##############################################################################
"""Database objects """Database objects
$Id: DB.py,v 1.39 2002/02/11 23:40:42 gvanrossum Exp $""" $Id: DB.py,v 1.40 2002/03/27 10:14:04 htrd Exp $"""
__version__='$Revision: 1.39 $'[11:-2] __version__='$Revision: 1.40 $'[11:-2]
import cPickle, cStringIO, sys, POSException, UndoLogCompatible import cPickle, cStringIO, sys, POSException, UndoLogCompatible
from Connection import Connection from Connection import Connection
...@@ -177,7 +177,7 @@ class DB(UndoLogCompatible.UndoLogCompatible): ...@@ -177,7 +177,7 @@ class DB(UndoLogCompatible.UndoLogCompatible):
def f(con, detail=detail, rc=sys.getrefcount, conn_no=conn_no): def f(con, detail=detail, rc=sys.getrefcount, conn_no=conn_no):
conn_no[0] = conn_no[0] + 1 conn_no[0] = conn_no[0] + 1
cn = conn_no[0] cn = conn_no[0]
for oid, ob in con._cache.items(): for oid, ob in con._cache_items():
id='' id=''
if hasattr(ob,'__dict__'): if hasattr(ob,'__dict__'):
d=ob.__dict__ d=ob.__dict__
...@@ -224,11 +224,21 @@ class DB(UndoLogCompatible.UndoLogCompatible): ...@@ -224,11 +224,21 @@ class DB(UndoLogCompatible.UndoLogCompatible):
def cacheSize(self): def cacheSize(self):
m=[0] m=[0]
def f(con, m=m): def f(con, m=m):
m[0]=m[0]+len(con._cache) m[0]=m[0]+con._cache.cache_non_ghost_count
self._connectionMap(f) self._connectionMap(f)
return m[0] return m[0]
def cacheDetailSize(self):
m=[]
def f(con, m=m):
m.append({'connection':repr(con),
'ngsize':con._cache.cache_non_ghost_count,
'size':len(con._cache)})
self._connectionMap(f)
m.sort()
return m
def close(self): self._storage.close() def close(self): self._storage.close()
def commitVersion(self, source, destination=''): def commitVersion(self, source, destination=''):
......
src/ZODB/cPersistence.c
View file @ 226ee421
...@@ -14,11 +14,21 @@ ...@@ -14,11 +14,21 @@
static char cPersistence_doc_string[] = static char cPersistence_doc_string[] =
"Defines Persistent mixin class for persistent objects.\n" "Defines Persistent mixin class for persistent objects.\n"
"\n" "\n"
"$Id: cPersistence.c,v 1.50 2002/03/08 18:36:13 jeremy Exp $\n"; "$Id: cPersistence.c,v 1.51 2002/03/27 10:14:04 htrd Exp $\n";
#include <string.h> #include <string.h>
#include "cPersistence.h" #include "cPersistence.h"
/* the layout of this struct is the same as the start of ccobject in cPickleCache.c */
struct ccobject_head_struct {
PyObject_HEAD
CPersistentRing ring_home;
int non_ghost_count;
};
#define HOME(O) ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->ring_home)) )
#define NON_GHOST_COUNT(O) ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->non_ghost_count)) )
#define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;} #define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
#define UNLESS(E) if(!(E)) #define UNLESS(E) if(!(E))
#define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V) #define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
...@@ -112,21 +122,82 @@ if(self->state < 0 && self->jar) \ ...@@ -112,21 +122,82 @@ if(self->state < 0 && self->jar) \
{ \ { \
PyObject *r; \ PyObject *r; \
\ \
int *count = NON_GHOST_COUNT(self); \
if(count) \
{ \
(*count)++; \
self->ring.next = HOME(self); \
self->ring.prev = HOME(self)->prev; \
HOME(self)->prev->next = &self->ring; \
HOME(self)->prev = &self->ring; \
Py_INCREF(self); \
} \
self->state=cPersistent_CHANGED_STATE; \ self->state=cPersistent_CHANGED_STATE; \
UNLESS(r=callmethod1(self->jar,py_setstate,(PyObject*)self)) \ UNLESS(r=callmethod1(self->jar,py_setstate,(PyObject*)self)) \
{ \ { \
self->state=cPersistent_GHOST_STATE; \ ghostify(self); \
return ER; \ return ER; \
} \ } \
self->state=cPersistent_UPTODATE_STATE; \ self->state=cPersistent_UPTODATE_STATE; \
Py_DECREF(r); \ Py_DECREF(r); \
} }
#define KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self) \
if(HOME(self) && self->state>=0) { \
self->ring.prev->next = self->ring.next; \
self->ring.next->prev = self->ring.prev; \
self->ring.next = HOME(self); \
self->ring.prev = HOME(self)->prev; \
HOME(self)->prev->next = &self->ring; \
HOME(self)->prev = &self->ring; }
/****************************************************************************/ /****************************************************************************/
staticforward PyExtensionClass Pertype; staticforward PyExtensionClass Pertype;
static void
accessed(cPersistentObject *self)
{
KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
}
static void
ghostify(cPersistentObject *self)
{
int *count;
count = NON_GHOST_COUNT(self);
if(count && (self->state>=0))
{
(*count)--;
self->ring.next->prev = self->ring.prev;
self->ring.prev->next = self->ring.next;
self->ring.prev = NULL;
self->ring.next = NULL;
self->state = cPersistent_GHOST_STATE;
Py_DECREF(self);
}
else
{
self->state = cPersistent_GHOST_STATE;
}
}
static void
deallocated(cPersistentObject *self)
{
if(self->state>=0) ghostify(self);
if(self->cache)
{
PyObject *v=PyObject_CallMethod(self->cache,"_oid_unreferenced","O",self->oid);
if(!v) PyErr_Clear(); /* and explode later */
Py_XDECREF(v);
}
Py_XDECREF(self->jar);
Py_XDECREF(self->oid);
}
static int static int
changed(cPersistentObject *self) changed(cPersistentObject *self)
{ {
...@@ -185,7 +256,7 @@ Per___changed__(cPersistentObject *self, PyObject *args) ...@@ -185,7 +256,7 @@ Per___changed__(cPersistentObject *self, PyObject *args)
static PyObject * static PyObject *
Per__p_deactivate(cPersistentObject *self, PyObject *args) Per__p_deactivate(cPersistentObject *self, PyObject *args)
{ {
PyObject *dict; PyObject *dict,*dict2=NULL;
#ifdef DEBUG_LOG #ifdef DEBUG_LOG
if (idebug_log < 0) call_debug("reinit",self); if (idebug_log < 0) call_debug("reinit",self);
...@@ -197,13 +268,22 @@ Per__p_deactivate(cPersistentObject *self, PyObject *args) ...@@ -197,13 +268,22 @@ Per__p_deactivate(cPersistentObject *self, PyObject *args)
if (self->state==cPersistent_UPTODATE_STATE && self->jar && if (self->state==cPersistent_UPTODATE_STATE && self->jar &&
HasInstDict(self) && (dict=INSTANCE_DICT(self))) HasInstDict(self) && (dict=INSTANCE_DICT(self)))
{ {
dict2 = PyDict_Copy(dict);
PyDict_Clear(dict); PyDict_Clear(dict);
/* Note that we need to set to ghost state unless we are /* Note that we need to set to ghost state unless we are
called directly. Methods that override this need to called directly. Methods that override this need to
do the same! */ do the same! */
self->state=cPersistent_GHOST_STATE; ghostify(self);
} }
/* need to delay releasing the last reference on instance attributes
until after we have finished accounting for losing our state */
if(dict2)
{
PyDict_Clear(dict2);
Py_DECREF(dict2);
}
Py_INCREF(Py_None); Py_INCREF(Py_None);
return Py_None; return Py_None;
} }
...@@ -333,8 +413,8 @@ Per_dealloc(cPersistentObject *self) ...@@ -333,8 +413,8 @@ Per_dealloc(cPersistentObject *self)
#ifdef DEBUG_LOG #ifdef DEBUG_LOG
if(idebug_log < 0) call_debug("del",self); if(idebug_log < 0) call_debug("del",self);
#endif #endif
Py_XDECREF(self->jar); deallocated(self);
Py_XDECREF(self->oid); Py_XDECREF(self->cache);
Py_DECREF(self->ob_type); Py_DECREF(self->ob_type);
PyObject_DEL(self); PyObject_DEL(self);
} }
...@@ -387,7 +467,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name, ...@@ -387,7 +467,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
{ {
UPDATE_STATE_IF_NECESSARY(self, NULL); UPDATE_STATE_IF_NECESSARY(self, NULL);
self->atime=((long)(time(NULL)/3))%65536; KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
if (self->serial[7]=='\0' && self->serial[6]=='\0' && if (self->serial[7]=='\0' && self->serial[6]=='\0' &&
self->serial[5]=='\0' && self->serial[4]=='\0' && self->serial[5]=='\0' && self->serial[4]=='\0' &&
...@@ -419,7 +499,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name, ...@@ -419,7 +499,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
{ {
UPDATE_STATE_IF_NECESSARY(self, NULL); UPDATE_STATE_IF_NECESSARY(self, NULL);
self->atime=((long)(time(NULL)/3))%65536; KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
} }
return getattrf((PyObject *)self, oname); return getattrf((PyObject *)self, oname);
...@@ -466,6 +546,21 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v, ...@@ -466,6 +546,21 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
{ {
if(name[3]=='o' && name[4]=='i' && name[5]=='d' && ! name[6]) if(name[3]=='o' && name[4]=='i' && name[5]=='d' && ! name[6])
{ {
if(HOME(self))
{
int result;
if(!v)
{
PyErr_SetString(PyExc_ValueError,"can not delete the oid of a cached object");
return -1;
}
if(PyObject_Cmp(self->oid,v,&result)<0) return -1;
if(result)
{
PyErr_SetString(PyExc_ValueError,"can not change the oid of a cached object");
return -1;
}
}
Py_XINCREF(v); Py_XINCREF(v);
ASSIGN(self->oid, v); ASSIGN(self->oid, v);
return 0; return 0;
...@@ -509,7 +604,6 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v, ...@@ -509,7 +604,6 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
v=PyObject_GetAttr(OBJECT(self), py__p_deactivate); v=PyObject_GetAttr(OBJECT(self), py__p_deactivate);
if (v) { ASSIGN(v, PyObject_CallObject(v, NULL)); } if (v) { ASSIGN(v, PyObject_CallObject(v, NULL)); }
if (v) { Py_DECREF(v); } if (v) { Py_DECREF(v); }
self->state=cPersistent_GHOST_STATE;
return 0; return 0;
} }
if (PyObject_IsTrue(v)) return changed(self); if (PyObject_IsTrue(v)) return changed(self);
...@@ -521,8 +615,7 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v, ...@@ -521,8 +615,7 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
{ {
UPDATE_STATE_IF_NECESSARY(self, -1); UPDATE_STATE_IF_NECESSARY(self, -1);
/* Record access times */ KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
self->atime=((long)(time(NULL)/3))%65536;
if((! (*name=='_' && name[1]=='v' && name[2]=='_')) if((! (*name=='_' && name[1]=='v' && name[2]=='_'))
&& (self->state != cPersistent_CHANGED_STATE && self->jar) && (self->state != cPersistent_CHANGED_STATE && self->jar)
...@@ -680,9 +773,11 @@ truecPersistenceCAPI = { ...@@ -680,9 +773,11 @@ truecPersistenceCAPI = {
(getattrofunc)Per_getattro, /*tp_getattr with object key*/ (getattrofunc)Per_getattro, /*tp_getattr with object key*/
(setattrofunc)Per_setattro, /*tp_setattr with object key*/ (setattrofunc)Per_setattro, /*tp_setattr with object key*/
changed, changed,
accessed,
ghostify,
deallocated,
(intfunctionwithpythonarg)Per_setstate, (intfunctionwithpythonarg)Per_setstate,
(pergetattr)Per_getattr, (pergetattr)Per_getattr,
(persetattr)_setattro,
}; };
void void
......
src/ZODB/cPersistence.h
View file @ 226ee421
...@@ -18,12 +18,21 @@ ...@@ -18,12 +18,21 @@
#include "ExtensionClass.h" #include "ExtensionClass.h"
#include <time.h> #include <time.h>
#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid; char serial[8]; unsigned short atime; signed char state; unsigned char reserved;
#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid, *cache; CPersistentRing ring; char serial[8]; signed char state; unsigned char reserved[3];
#define cPersistent_GHOST_STATE -1 #define cPersistent_GHOST_STATE -1
#define cPersistent_UPTODATE_STATE 0 #define cPersistent_UPTODATE_STATE 0
#define cPersistent_CHANGED_STATE 1 #define cPersistent_CHANGED_STATE 1
#define cPersistent_STICKY_STATE 2 #define cPersistent_STICKY_STATE 2
struct ccobject_head_struct;
typedef struct CPersistentRing_struct
{
struct CPersistentRing_struct *prev;
struct CPersistentRing_struct *next;
} CPersistentRing;
typedef struct { typedef struct {
cPersistent_HEAD cPersistent_HEAD
} cPersistentObject; } cPersistentObject;
...@@ -36,6 +45,9 @@ typedef struct { ...@@ -36,6 +45,9 @@ typedef struct {
getattrofunc getattro; getattrofunc getattro;
setattrofunc setattro; setattrofunc setattro;
int (*changed)(cPersistentObject*); int (*changed)(cPersistentObject*);
void (*accessed)(cPersistentObject*);
void (*ghostify)(cPersistentObject*);
void (*deallocated)(cPersistentObject*);
int (*setstate)(PyObject*); int (*setstate)(PyObject*);
pergetattr pergetattro; pergetattr pergetattro;
persetattr persetattro; persetattr persetattro;
...@@ -59,11 +71,13 @@ static cPersistenceCAPIstruct *cPersistenceCAPI; ...@@ -59,11 +71,13 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
#define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O))) #define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O)))
#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE)) #define PER_GHOSTIFY(O) (cPersistenceCAPI->ghostify((cPersistentObject*)(O)))
#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
#define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE)) #define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
#define PER_DEL(O) Py_XDECREF((O)->jar); Py_XDECREF((O)->oid); #define PER_DEL(O) (cPersistenceCAPI->deallocated((cPersistentObject*)(O)))
#define PER_USE(O) \ #define PER_USE(O) \
(((O)->state != cPersistent_GHOST_STATE \ (((O)->state != cPersistent_GHOST_STATE \
...@@ -71,7 +85,7 @@ static cPersistenceCAPIstruct *cPersistenceCAPI; ...@@ -71,7 +85,7 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
? (((O)->state==cPersistent_UPTODATE_STATE) \ ? (((O)->state==cPersistent_UPTODATE_STATE) \
? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0) ? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0)
#define PER_ACCESSED(O) ((O)->atime=((long)(time(NULL)/3))%65536) #define PER_ACCESSED(O) (cPersistenceCAPI->accessed((cPersistentObject*)(O)))
#endif #endif
......
src/ZODB/cPickleCache.c
View file @ 226ee421
...@@ -2,448 +2,531 @@ ...@@ -2,448 +2,531 @@
Copyright (c) 2001, 2002 Zope Corporation and Contributors. Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved. All Rights Reserved.
This software is subject to the provisions of the Zope Public License, This software is subject to the provisions of the Zope Public License,
Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE FOR A PARTICULAR PURPOSE
****************************************************************************/ ****************************************************************************/
static char cPickleCache_doc_string[] =
static char cPickleCache_doc_string[] =
"Defines the PickleCache used by ZODB Connection objects.\n" "Defines the PickleCache used by ZODB Connection objects.\n"
"\n" "\n"
"$Id: cPickleCache.c,v 1.41 2002/03/08 18:36:14 jeremy Exp $\n"; "$Id: cPickleCache.c,v 1.42 2002/03/27 10:14:04 htrd Exp $\n";
/* Compute the current time in the units and range used for peristent #define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
objects. */ #define UNLESS(E) if(!(E))
#define PER_TIME() ((long)(time(NULL) / 3)) % 65536 #define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
#define OBJECT(O) ((PyObject*)O)
#define DONT_USE_CPERSISTENCECAPI #define DONT_USE_CPERSISTENCECAPI
#include "cPersistence.h" #include "cPersistence.h"
#include <time.h> #include <time.h>
#include <stddef.h>
#undef Py_FindMethod #undef Py_FindMethod
static PyObject *py_reload, *py__p_jar, *py__p_changed;
typedef struct { static PyObject *py__p_oid, *py_reload, *py__p_jar, *py__p_changed;
PyObject_HEAD
PyObject *data;
PyObject *jar;
PyObject *setklassstate;
int position;
int cache_size;
int cache_age;
/* Cache statistics */
int sum_deal;
int sum_deac;
double sum_age;
int n, na;
time_t last_check; /* Time of last gc */
double mean_age;
double mean_deal;
double mean_deac;
double df, dfa; /* Degees of freedom for above stats */
} ccobject;
#define WEIGHTING_PERIOD 600 /* define this for extra debugging checks, and lousy performance */
#define MUCH_RING_CHECKING 1
/* /* Do we want 'engine noise'.... abstract debugging output useful for
How to compute weighted means? visualizing cache behavior */
#if 0
#define ENGINE_NOISE(A) printf(A)
#else
#define ENGINE_NOISE(A) ((void)A)
#endif
Assume we have two means, a current mean, M, and a mean as of some /* the layout of this struct is the same as the start of ccobject_head in cPersistence.c */
time d seconds in the past, Md. The means have effective degrees typedef struct {
of freedom, N, and Nd. Where Nd is adjusted by d is some fashion. PyObject_HEAD
The combined mean is (M*N+Md*Nd)/(N+Nd). The degrees of freedom CPersistentRing ring_home;
of the combined mean, Nc, is N+Nd. Nd is computed by weighting int non_ghost_count;
an old degree of freedom with the weight: I/(I+d), where I is some int klass_count;
suitably chosen constant, which we will call a "weighting period". PyObject *data;
PyObject *jar;
*/ PyObject *setklassstate;
int cache_size;
int ring_lock;
int cache_drain_resistance;
} ccobject;
staticforward PyTypeObject Cctype; staticforward PyTypeObject Cctype;
staticforward int present_in_ring(ccobject *self,CPersistentRing *target);
staticforward int check_ring(ccobject *self,const char *context);
staticforward int cc_ass_sub(ccobject *self, PyObject *key, PyObject *v);
/* ---------------------------------------------------------------- */ /* ---------------------------------------------------------------- */
static int static PyObject *object_from_oid(ccobject *self,PyObject *key)
gc_item(ccobject *self, PyObject *key, PyObject *v, long now, int dt) /* somewhat of a replacement for PyDict_GetItem(self->data....
however this returns a *new* reference */
{ {
if (!(v && key)) PyObject *v = PyDict_GetItem(self->data, key);
return 0; if(!v) return NULL;
self->n++;
/* If there is at most one reference to this object, then the
cache has the only reference. It can be removed. */
if (v->ob_refcnt <= 1) {
self->sum_deal++;
/* XXX The fact that this works will iterating over
self->data with PyDict_Next() is an accident of the
current Python dictionary implementation. */
return PyDict_DelItem(self->data, key);
}
if (dt >= 0 && Py_INCREF(v);
(!PyExtensionClass_Check(v)) &&
((cPersistentObject*)v)->jar == self->jar /* I'm paranoid */ && return v;
((cPersistentObject*)v)->state == cPersistent_UPTODATE_STATE) {
now -= ((cPersistentObject*)v)->atime;
if (now < 0)
now += 65536;
self->na++;
self->sum_age += now;
if (now > dt) {
/* We have a cPersistent object that hasn't been used in
a while. Reinitialize it, hopefully freeing it's
state.
*/
self->sum_deac++;
if (PyObject_SetAttr(v, py__p_changed, Py_None) < 0)
PyErr_Clear();
}
}
return 0;
} }
static void static cPersistentObject *object_from_ring(ccobject *self,CPersistentRing *here,const char *context)
update_stats(ccobject *self, time_t now)
{ {
double d, deal, deac; /* Given a position in the LRU ring, return a borrowed
reference to the object at that point in the ring. The caller is
d = now - self->last_check; responsible for ensuring that this ring position really does
if(d < 1) correspond to a persistent object, although the debugging
return; version will double-check this. */
self->df *= WEIGHTING_PERIOD / (WEIGHTING_PERIOD + d); PyObject *object = (PyObject *)(((char *)here)-offsetof(cPersistentObject,ring));
self->dfa *= WEIGHTING_PERIOD / (WEIGHTING_PERIOD + d);
#ifdef MUCH_RING_CHECKING
self->mean_age = ((self->mean_age * self->dfa + self->sum_age)/ if(!PyExtensionInstance_Check(object))
(self->dfa + self->na)) * 3; {
self->sum_age = 0; PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered non-ExtensionClass object in %s",context);
return NULL;
deac = self->sum_deac / d; }
self->sum_deac = 0; if(!(((PyExtensionClass*)(object->ob_type))->class_flags & PERSISTENT_TYPE_FLAG))
self->mean_deac = ((self->mean_deac * self->dfa+deac)/ {
(self->dfa + self->na)); PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered non-persistent object in %s",context);
self->sum_deac = 0; return NULL;
}
self->dfa += self->na; if(((cPersistentObject*)object)->jar!=self->jar)
self->na = 0; {
PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered object from a different jar in %s",context);
deal=self->sum_deal/d; return NULL;
self->sum_deal = 0; }
self->mean_deal = ((self->mean_deal * self->df + deal)/ if(((cPersistentObject*)object)->cache!=(PyObject *)self)
(self->df +self->n)); {
self->sum_deal = 0; PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered broken ring in %s",context);
return NULL;
self->df += self->n; }
self->n = 0; #endif
return (cPersistentObject *)object;
self->last_check = now;
} }
static int static int
check_size(ccobject *self) scan_gc_items(ccobject *self,int target)
{ {
if (self->cache_size < 1) cPersistentObject *object;
return 0; int error;
return PyDict_Size(self->data); CPersistentRing placeholder;
CPersistentRing *here = self->ring_home.next;
#ifdef MUCH_RING_CHECKING
int safety_counter = self->cache_size*10;
if(safety_counter<10000) safety_counter = 10000;
#endif
while(1)
{
if(check_ring(self,"mid-gc")) return -1;
#ifdef MUCH_RING_CHECKING
if(!safety_counter--)
{
/* This loop has been running for a very long time.
It is possible that someone loaded a very large number of objects,
and now wants us to blow them all away. However it may
also indicate a logic error. If the loop has been running this
long then you really have to doubt it will ever terminate.
In the MUCH_RING_CHECKING build we prefer to raise an exception
here */
PyErr_SetString(PyExc_RuntimeError,"scan_gc_items safety counter exceeded");
return -1;
}
if(!present_in_ring(self,here))
{
/* Our current working position is no longer in the ring. Thats bad. */
PyErr_SetString(PyExc_RuntimeError,"working position fell out the ring, in scan_gc_items");
return -1;
}
#endif
if(here==&self->ring_home)
{
/* back to the home position. stop looking */
return 0;
}
/* At this point we know that the ring only contains nodes from
persistent objects, plus our own home node. We can safely
assume this is a persistent object now we know it is not the home */
object = object_from_ring(self,here,"scan_gc_items");
if(!object) return -1;
if(self->non_ghost_count<=target)
{
/* we are small enough */
return 0;
}
else if(object->state==cPersistent_UPTODATE_STATE)
{
/* deactivate it. This is the main memory saver. */
ENGINE_NOISE("G");
/* add a placeholder */
placeholder.next = here->next;
placeholder.prev = here;
here->next->prev = &placeholder;
here->next = &placeholder;
error = PyObject_SetAttr((PyObject *)object,py__p_changed,Py_None);
/* unlink the placeholder */
placeholder.next->prev=placeholder.prev;
placeholder.prev->next=placeholder.next;
here = placeholder.next;
if(error)
return -1; /* problem */
}
else
{
ENGINE_NOISE(".");
here = here->next;
}
}
} }
static int
gc_all_items(ccobject *self, int now, int dt) static PyObject *
lockgc(ccobject *self,int target_size)
{ {
PyObject *key, *v; if(self->ring_lock)
int i; {
Py_INCREF(Py_None);
return Py_None;
}
for(i = 0; PyDict_Next(self->data, &i, &key, &v); ) if(check_ring(self,"pre-gc")) return NULL;
if (gc_item(self, key, v, now, dt) < 0) ENGINE_NOISE("<");
return -1; self->ring_lock = 1;
return 0; if(scan_gc_items(self,target_size))
{
self->ring_lock = 0;
return NULL;
}
self->ring_lock = 0;
ENGINE_NOISE(">\n");
if(check_ring(self,"post-gc")) return NULL;
Py_INCREF(Py_None);
return Py_None;
} }
static int static PyObject *
fullgc(ccobject *self, int dt) cc_incrgc(ccobject *self, PyObject *args)
{ {
long now; int n=1;
if (check_size(self) <= 0) int starting_size = self->non_ghost_count;
return 0;
now = PER_TIME(); int target_size = self->cache_size;
dt /= 3;
if (gc_all_items(self, now, dt) < 0) if(self->cache_drain_resistance>=1)
return -1; {
self->position = 0; /* This cache will gradually drain down to a small size. Check
a (small) number of objects proportional to the current size */
if (now - self->last_check > 1) int target_size_2 = starting_size - 1 - starting_size/self->cache_drain_resistance;
update_stats(self, now); if(target_size_2<target_size)
target_size = target_size_2;
return 0;
}
static int
reallyfullgc(ccobject *self, int dt)
{
int l, last;
time_t now;
last = check_size(self);
if (last <= 0)
return 0;
now = PER_TIME();
/* Units are 3 seconds */
dt /= 3;
/* First time through should get refcounts to 1 */
if (gc_all_items(self, now, dt) < 0)
return -1;
l = PyDict_Size(self->data);
if (l < 0)
return -1;
/* Now continue to collect until the size of the cache stops
decreasing. */
while (l < last) {
if (gc_all_items(self, now, dt) < 0)
return -1;
last = l;
l = PyDict_Size(self->data);
if (l < 0)
return -1;
} }
if (now - self->last_check > 1)
update_stats(self, now);
self->position = 0;
return 0;
}
static int UNLESS (PyArg_ParseTuple(args, "|i",&n)) return NULL;
maybegc(ccobject *self, PyObject *thisv)
{
int n, s, size, dt;
long now;
PyObject *key=0, *v=0;
s = check_size(self);
if (s <= 0)
return 0;
now = PER_TIME();
size = self->cache_size;
self->cache_size = 0;
/* Decide how many objects to look at */
n = (s - size) / 10;
if (n < 3)
n = 3;
/* Decide how much time to give them before deactivating them */
s = 8 * size / s;
if (s > 100)
s = 100;
dt = (long)(self->cache_age * (0.2 + 0.1 * s));
/* Units are 3 seconds */
dt /= 3;
while (--n >= 0) {
if (PyDict_Next(self->data, &(self->position), &key, &v)) {
if (v != thisv && gc_item(self, key, v, now, dt) < 0) {
self->cache_size=size;
return -1;
}
}
else
self->position = 0;
}
self->cache_size = size;
if (now - self->last_check > 1)
update_stats(self, now);
return 0; return lockgc(self,target_size);
} }
static PyObject * static PyObject *
cc_full_sweep(ccobject *self, PyObject *args) cc_full_sweep(ccobject *self, PyObject *args)
{ {
int dt = self->cache_age; int dt=0;
if (!PyArg_ParseTuple(args, "|i:full_sweep", &dt)) UNLESS(PyArg_ParseTuple(args, "|i", &dt)) return NULL;
return NULL; return lockgc(self,0);
if (dt < -1) {
PyErr_SetString(PyExc_ValueError, "age must be >= -1");
return NULL;
}
if (fullgc(self, dt) == -1)
return NULL;
Py_INCREF(Py_None);
return Py_None;
} }
static PyObject * static PyObject *
cc_reallyfull_sweep(ccobject *self, PyObject *args) cc_reallyfull_sweep(ccobject *self, PyObject *args)
{ {
int dt = self->cache_age; int dt=0;
if (!PyArg_ParseTuple(args, "|i:minimize", &dt)) UNLESS(PyArg_ParseTuple(args, "|i", &dt)) return NULL;
return NULL; return lockgc(self,0);
if (dt < -1) {
PyErr_SetString(PyExc_ValueError, "age must be >= -1");
return NULL;
}
if (reallyfullgc(self, dt) == -1)
return NULL;
Py_INCREF(Py_None);
return Py_None;
} }
static PyObject * static void
cc_incrgc(ccobject *self, PyObject *args) _invalidate(ccobject *self, PyObject *key)
{ {
int n = 1; PyObject *v=object_from_oid(self, key);
if (!PyArg_ParseTuple(args, "|i:incrgr", &n)) if(!v)
return NULL; {
/* shouldnt this be an error? for now Ill follow Jims lead */
PyErr_Clear();
}
else
{
if (PyExtensionClass_Check(v))
{
if(v->ob_refcnt <= 1)
{
self->klass_count--;
if (PyDict_DelItem(self->data, key) < 0)
PyErr_Clear();
}
else
{
v=PyObject_CallFunction(self->setklassstate,
"O", v);
if (v) Py_DECREF(v);
else PyErr_Clear();
}
}
else
{
if(PyObject_DelAttr(v,py__p_changed) < 0)
PyErr_Clear();
}
Py_DECREF(v);
}
}
for (; --n >= 0;) static PyObject *
if (maybegc(self, NULL) < 0) cc_invalidate(ccobject *self, PyObject *args)
return NULL; {
PyObject *inv, *key, *v;
int i;
if (PyArg_ParseTuple(args, "O!", &PyDict_Type, &inv)) {
for (i=0; PyDict_Next(inv, &i, &key, &v); )
if (key==Py_None)
{ /* Eek some nitwit invalidated everything! */
for (i=0; PyDict_Next(self->data, &i, &key, &v); )
_invalidate(self, key);
break;
}
else
_invalidate(self, key);
PyDict_Clear(inv);
}
else {
PyErr_Clear();
UNLESS (PyArg_ParseTuple(args, "O", &inv)) return NULL;
if (PyString_Check(inv))
_invalidate(self, inv);
else if (inv==Py_None) /* All */
for (i=0; PyDict_Next(self->data, &i, &key, &v); )
_invalidate(self, key);
else {
int l;
PyErr_Clear();
if ((l=PyObject_Length(inv)) < 0) return NULL;
for(i=l; --i >= 0; )
{
UNLESS (key=PySequence_GetItem(inv, i)) return NULL;
_invalidate(self, key);
Py_DECREF(key);
}
PySequence_DelSlice(inv, 0, l);
}
}
Py_INCREF(Py_None); Py_INCREF(Py_None);
return Py_None; return Py_None;
} }
static void
_invalidate(ccobject *self, PyObject *key) static PyObject *
cc_get(ccobject *self, PyObject *args)
{ {
PyObject *v = PyDict_GetItem(self->data, key); PyObject *r, *key, *d=0;
if (!v) UNLESS (PyArg_ParseTuple(args,"O|O", &key, &d)) return NULL;
return;
if (PyExtensionClass_Check(v)) UNLESS (r=(PyObject *)object_from_oid(self, key))
if (v->ob_refcnt <= 1) { {
self->sum_deal++; if (d)
if (PyDict_DelItem(self->data, key) < 0) {
PyErr_Clear(); PyErr_Clear();
} else { r=d;
PyObject *t = PyTuple_New(1); Py_INCREF(r);
if (t) { }
PyTuple_SET_ITEM(t, 0, v); else
v = PyObject_CallObject(self->setklassstate, t); {
/* Set tuple element to NULL so that deallocating the PyErr_SetObject(PyExc_KeyError, key);
tuple does not decref t. return NULL;
*/
PyTuple_SET_ITEM(t, 0, NULL);
Py_DECREF(t);
} else
v = t;
if (v)
Py_DECREF(v);
else
PyErr_Clear();
} }
else if (PyObject_DelAttr(v, py__p_changed) < 0) }
PyErr_Clear();
return r;
} }
static void static PyObject *
_invalidate_all(ccobject *self) cc_klass_items(ccobject *self, PyObject *args)
{ {
PyObject *key, *v; PyObject *l,*k,*v;
int i; int p = 0;
if(!PyArg_ParseTuple(args,"")) return NULL;
l = PyList_New(0);
if(!l) return NULL;
while(PyDict_Next(self->data, &p, &k, &v))
{
if(PyExtensionClass_Check(v))
{
v=PyObject_CallMethod(l,"append","((OO))",k,v);
if(!v)
{
Py_DECREF(l);
return NULL;
}
}
}
for (i = 0; PyDict_Next(self->data, &i, &key, &v); ) return l;
_invalidate(self, key);
} }
static PyObject * static PyObject *
cc_invalidate(ccobject *self, PyObject *args) cc_lru_items(ccobject *self, PyObject *args)
{ {
PyObject *inv, *key, *v; PyObject *l;
int i; CPersistentRing *here;
if (!PyArg_ParseTuple(args, "O:invalidate", &inv)) if(!PyArg_ParseTuple(args,"")) return NULL;
return NULL;
if (PyDict_Check(inv)) { if(self->ring_lock)
for (i = 0; PyDict_Next(inv, &i, &key, &v); ) {
if (key == Py_None) { PyErr_SetString(PyExc_ValueError,".lru_items() is unavailable during garbage collection");
/* Eek some nitwit invalidated everything! */ return NULL;
_invalidate_all(self);
break;
}
else
_invalidate(self, key);
PyDict_Clear(inv);
} else if (PyString_Check(inv))
_invalidate(self, inv);
else if (inv == Py_None) /* All */
_invalidate_all(self);
else {
int l = PyObject_Length(inv);
if (l < 0)
return NULL;
for (i = l; --i >= 0; ) {
key = PySequence_GetItem(inv, i);
if (!key)
return NULL;
_invalidate(self, key);
Py_DECREF(key);
}
PySequence_DelSlice(inv, 0, l);
} }
Py_INCREF(Py_None); if(check_ring(self,"pre-cc_items")) return NULL;
return Py_None;
l = PyList_New(0);
if(!l) return NULL;
here = self->ring_home.next;
while(here!=&self->ring_home)
{
cPersistentObject *object = object_from_ring(self,here,"cc_items");
PyObject *v;
if(!object)
{
Py_DECREF(l);
return NULL;
}
v=PyObject_CallMethod(l,"append","((OO))",object->oid,object);
if(!v)
{
Py_DECREF(l);
return NULL;
}
Py_DECREF(v);
here = here->next;
}
return l;
} }
static PyObject * static PyObject *
cc_get(ccobject *self, PyObject *args) cc_oid_unreferenced(ccobject *self, PyObject *args)
{ {
PyObject *r, *key, *d = NULL; PyObject *oid,*v;
if(!PyArg_ParseTuple(args,"O",&oid)) return NULL;
if (!PyArg_ParseTuple(args, "O|O:get", &key, &d)) v = PyDict_GetItem(self->data, oid);
return NULL; if(!v) return NULL;
r = PyDict_GetItem(self->data, key); if(v->ob_refcnt)
if (!r) { {
if (d) PyErr_Format(PyExc_ValueError,"object has reference count of %d, should be zero",v->ob_refcnt);
r = d; return NULL;
else {
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
}
} }
Py_INCREF(r); /* Need to be very hairy here because a dictionary is about
return r; to decref an already deleted object */
#ifdef Py_TRACE_REFS
#error "this code path has not been tested - Toby Dickenson"
_Py_NewReference(v);
/* it may be a problem that v->ob_type is still NULL? */
#else
Py_INCREF(v);
#endif
if(v->ob_refcnt!=1)
{
PyErr_SetString(PyExc_ValueError,"refcount is not 1 after resurrection");
return NULL;
}
/* return the stolen reference */
Py_INCREF(v);
PyDict_DelItem(self->data, oid);
if(v->ob_refcnt!=1)
{
PyErr_SetString(PyExc_ValueError,"refcount is not 1 after removal from dict");
return NULL;
}
/* undo the temporary resurrection */
#ifdef Py_TRACE_REFS
_Py_ForgetReference(v);
#else
v->ob_refcnt=0;
#endif
Py_INCREF(Py_None);
return Py_None;
} }
static struct PyMethodDef cc_methods[] = { static struct PyMethodDef cc_methods[] = {
{"_oid_unreferenced", (PyCFunction)cc_oid_unreferenced, METH_VARARGS,
NULL
},
{"lru_items", (PyCFunction)cc_lru_items, METH_VARARGS,
"List (oid, object) pairs from the lru list, as 2-tuples.\n"
},
{"klass_items", (PyCFunction)cc_klass_items, METH_VARARGS,
"List (oid, object) pairs of cached persistent classes.\n"
},
{"full_sweep", (PyCFunction)cc_full_sweep, METH_VARARGS, {"full_sweep", (PyCFunction)cc_full_sweep, METH_VARARGS,
"full_sweep([age]) -- Perform a full sweep of the cache\n\n" "full_sweep([age]) -- Perform a full sweep of the cache\n\n"
"Make a single pass through the cache, removing any objects that are no\n" "Make a single pass through the cache, removing any objects that are no\n"
"longer referenced, and deactivating objects that have not been\n" "longer referenced, and deactivating enough objects to bring\n"
"accessed in the number of seconds given by 'age'. " "the cache under its size limit\n"
"'age defaults to the cache age.\n" "The optional 'age' parameter is ignored.\n"
}, },
{"minimize", (PyCFunction)cc_reallyfull_sweep, METH_VARARGS, {"minimize", (PyCFunction)cc_reallyfull_sweep, METH_VARARGS,
"minimize([age]) -- Remove as many objects as possible\n\n" "minimize([age]) -- Remove as many objects as possible\n\n"
"Make multiple passes through the cache, removing any objects that are no\n" "Make multiple passes through the cache, removing any objects that are no\n"
"longer referenced, and deactivating objects that have not been\n" "longer referenced, and deactivating enough objects to bring the"
"accessed in the number of seconds given by 'age'. 'age defaults to 0.\n" " cache under its size limit\n"
"The option 'age' parameter is ignored.\n"
}, },
{"incrgc", (PyCFunction)cc_incrgc, METH_VARARGS, {"incrgc", (PyCFunction)cc_incrgc, METH_VARARGS,
"incrgc() -- Perform incremental garbage collection"}, "incrgc([n]) -- Perform incremental garbage collection\n\n"
"Some other implementations support an optional parameter 'n' which\n"
"indicates a repetition count; this value is ignored.\n"},
{"invalidate", (PyCFunction)cc_invalidate, METH_VARARGS, {"invalidate", (PyCFunction)cc_invalidate, METH_VARARGS,
"invalidate(oids) -- invalidate one, many, or all ids"}, "invalidate(oids) -- invalidate one, many, or all ids"},
{"get", (PyCFunction)cc_get, METH_VARARGS, {"get", (PyCFunction)cc_get, METH_VARARGS,
...@@ -454,115 +537,102 @@ static struct PyMethodDef cc_methods[] = { ...@@ -454,115 +537,102 @@ static struct PyMethodDef cc_methods[] = {
static ccobject * static ccobject *
newccobject(PyObject *jar, int cache_size, int cache_age) newccobject(PyObject *jar, int cache_size, int cache_age)
{ {
ccobject *self; ccobject *self;
self = PyObject_NEW(ccobject, &Cctype); UNLESS(self = PyObject_NEW(ccobject, &Cctype)) return NULL;
if (!self) self->setklassstate=self->jar=NULL;
if((self->data=PyDict_New()))
{
self->jar=jar;
Py_INCREF(jar);
UNLESS (self->setklassstate=PyObject_GetAttrString(jar, "setklassstate"))
return NULL; return NULL;
self->setklassstate = self->jar = NULL; self->cache_size=cache_size;
self->data = PyDict_New(); self->non_ghost_count=0;
if (self->data) { self->klass_count=0;
self->jar=jar; self->cache_drain_resistance=0;
Py_INCREF(jar); self->ring_lock=0;
self->setklassstate = PyObject_GetAttrString(jar, "setklassstate"); self->ring_home.next = &self->ring_home;
if (!self->setklassstate) { self->ring_home.prev = &self->ring_home;
Py_DECREF(jar); return self;
Py_DECREF(self->data);
goto error;
}
self->position = 0;
self->cache_size = cache_size;
self->cache_age = cache_age < 1 ? 1 : cache_age;
self->sum_deal = 0;
self->sum_deac = 0;
self->sum_age = 0;
self->mean_deal = 0;
self->mean_deac = 0;
self->mean_age = 0;
self->df = 1;
self->dfa = 1;
self->n = 0;
self->na = 0;
self->last_check = time(NULL);
return self;
} }
error: Py_DECREF(self);
Py_DECREF(self); return NULL;
return NULL;
} }
static void static void
cc_dealloc(ccobject *self) cc_dealloc(ccobject *self)
{ {
Py_XDECREF(self->data); Py_XDECREF(self->data);
Py_XDECREF(self->jar); Py_XDECREF(self->jar);
Py_XDECREF(self->setklassstate); Py_XDECREF(self->setklassstate);
PyObject_DEL(self); PyMem_DEL(self);
} }
static PyObject * static PyObject *
cc_getattr(ccobject *self, char *name) cc_getattr(ccobject *self, char *name)
{ {
PyObject *r; PyObject *r;
if (*name == 'c') { if(check_ring(self,"getattr")) return NULL;
if(strcmp(name, "cache_age") == 0)
return PyInt_FromLong(self->cache_age); if(*name=='c')
if(strcmp(name, "cache_size") == 0) {
return PyInt_FromLong(self->cache_size); if(strcmp(name,"cache_age")==0)
if(strcmp(name, "cache_mean_age") == 0) return PyInt_FromLong(0); /* this cache does not use this value */
return PyFloat_FromDouble(self->mean_age); if(strcmp(name,"cache_size")==0)
if(strcmp(name, "cache_mean_deal") == 0) return PyInt_FromLong(self->cache_size);
return PyFloat_FromDouble(self->mean_deal); if(strcmp(name,"cache_drain_resistance")==0)
if(strcmp(name, "cache_mean_deac") == 0) return PyInt_FromLong(self->cache_drain_resistance);
return PyFloat_FromDouble(self->mean_deac); if(strcmp(name,"cache_non_ghost_count")==0)
if(strcmp(name, "cache_df") == 0) return PyInt_FromLong(self->non_ghost_count);
return PyFloat_FromDouble(self->df); if(strcmp(name,"cache_klass_count")==0)
if(strcmp(name, "cache_dfa") == 0) return PyInt_FromLong(self->klass_count);
return PyFloat_FromDouble(self->dfa); if(strcmp(name,"cache_data")==0)
if(strcmp(name, "cache_last_gc_time") == 0) {
return PyFloat_FromDouble(self->last_check); /* now a copy of our data; the ring is too fragile */
if(strcmp(name, "cache_data") == 0) { return PyDict_Copy(self->data);
Py_INCREF(self->data);
return self->data;
} }
} }
if ((strcmp(name, "has_key") == 0) if((*name=='h' && strcmp(name, "has_key")==0) ||
|| (strcmp(name, "items") == 0) (*name=='i' && strcmp(name, "items")==0) ||
|| (strcmp(name, "keys") == 0)) (*name=='k' && strcmp(name, "keys")==0)
return PyObject_GetAttrString(self->data, name); )
return PyObject_GetAttrString(self->data, name);
r = Py_FindMethod(cc_methods, (PyObject *)self, name);
if (!r) { if((r=Py_FindMethod(cc_methods, (PyObject *)self, name)))
PyErr_Clear();
return PyObject_GetAttrString(self->data, name);
}
return r; return r;
PyErr_Clear();
return PyObject_GetAttrString(self->data, name);
} }
static int static int
cc_setattr(ccobject *self, char *name, PyObject *value) cc_setattr(ccobject *self, char *name, PyObject *value)
{ {
if (value) { if(value)
{
int v; int v;
if (strcmp(name, "cache_age") == 0) { if(strcmp(name,"cache_age")==0)
v = PyInt_AsLong(value); {
if (v == -1 && PyErr_Occurred()) /* this cache doesnt use the age */
return -1;
if (v > 0)
self->cache_age = v;
return 0; return 0;
} }
if (strcmp(name, "cache_size") == 0) { if(strcmp(name,"cache_size")==0)
v = PyInt_AsLong(value); {
if (v == -1 && PyErr_Occurred()) UNLESS(PyArg_Parse(value,"i",&v)) return -1;
return -1; self->cache_size=v;
self->cache_size = v;
return 0; return 0;
} }
}
if(strcmp(name,"cache_drain_resistance")==0)
{
UNLESS(PyArg_Parse(value,"i",&v)) return -1;
self->cache_drain_resistance=v;
return 0;
}
}
PyErr_SetString(PyExc_AttributeError, name); PyErr_SetString(PyExc_AttributeError, name);
return -1; return -1;
} }
...@@ -570,7 +640,7 @@ cc_setattr(ccobject *self, char *name, PyObject *value) ...@@ -570,7 +640,7 @@ cc_setattr(ccobject *self, char *name, PyObject *value)
static int static int
cc_length(ccobject *self) cc_length(ccobject *self)
{ {
return PyDict_Size(self->data); return PyObject_Length(self->data);
} }
static PyObject * static PyObject *
...@@ -578,39 +648,245 @@ cc_subscript(ccobject *self, PyObject *key) ...@@ -578,39 +648,245 @@ cc_subscript(ccobject *self, PyObject *key)
{ {
PyObject *r; PyObject *r;
r = PyDict_GetItem(self->data, key); if(check_ring(self,"__getitem__")) return NULL;
if (!r) {
PyErr_SetObject(PyExc_KeyError, key); UNLESS (r=(PyObject *)object_from_oid(self, key))
return NULL; {
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
} }
Py_INCREF(r);
return r; return r;
} }
static int static int
cc_ass_sub(ccobject *self, PyObject *key, PyObject *v) cc_ass_sub(ccobject *self, PyObject *key, PyObject *v)
{ {
if (v) { int result;
if (PyExtensionClass_Check(v) if(v)
|| {
(PyExtensionInstance_Check(v) if( ( PyExtensionInstance_Check(v) &&
&& (((PyExtensionClass*)(v->ob_type))->class_flags & PERSISTENT_TYPE_FLAG) &&
(((PyExtensionClass*)(v->ob_type))->class_flags (v->ob_type->tp_basicsize >= sizeof(cPersistentObject))
& PERSISTENT_TYPE_FLAG) )
&& ||
(v->ob_type->tp_basicsize >= sizeof(cPersistentObject)) PyExtensionClass_Check(v)
) )
) {
return PyDict_SetItem(self->data, key, v); PyObject *oid = PyObject_GetAttr(v,py__p_oid);
PyObject *object_again;
PyErr_SetString(PyExc_ValueError, if(!oid)
"Cache values must be persistent objects or classes."); {
return -1; return -1;
}
if(PyObject_Cmp(key,oid,&result))
{
Py_DECREF(oid);
return -1;
}
Py_DECREF(oid);
if(result)
{
PyErr_SetString(PyExc_ValueError,"key must be the same as the object's oid attribute");
return -1;
}
object_again = object_from_oid(self, key);
if(object_again)
{
if(object_again!=v)
{
Py_DECREF(object_again);
PyErr_SetString(PyExc_ValueError,"Can not re-register object under a different oid");
return -1;
}
else
{
/* re-register under the same oid - no work needed */
Py_DECREF(object_again);
return 0;
}
}
if(PyExtensionClass_Check(v))
{
if(PyDict_SetItem(self->data, key, v)) return -1;
self->klass_count++;
return 0;
}
else
{
if(((cPersistentObject*)v)->cache)
{
if(((cPersistentObject*)v)->cache==(PyObject *)self)
{
/* This object is already one of ours, which is ok.
It would be very strange if someone was trying to register the
same object under a different key */
}
else
{
/* This object is already in a different cache. */
PyErr_SetString(PyExc_ValueError, "Cache values may only be in one cache.");
return -1;
}
}
if(check_ring(self,"pre-setitem")) return -1;
if(PyDict_SetItem(self->data, key, v)) return -1;
Py_INCREF(self);
((cPersistentObject*)v)->cache = (PyObject *)self;
if(((cPersistentObject*)v)->state>=0)
{
/* insert this non-ghost object into the ring just behind the home position */
self->non_ghost_count++;
((cPersistentObject*)v)->ring.next = &self->ring_home;
((cPersistentObject*)v)->ring.prev = self->ring_home.prev;
self->ring_home.prev->next = &((cPersistentObject*)v)->ring;
self->ring_home.prev = &((cPersistentObject*)v)->ring;
}
else
{
/* steal a reference from the dictionary; ghosts have a weak reference */
Py_DECREF(v);
}
if(check_ring(self,"post-setitem")) return -1;
return 0;
}
}
else
{
PyErr_SetString(PyExc_ValueError, "Cache values must be persistent objects.");
return -1;
}
}
else
{
/* unlink this item from the ring */
if(check_ring(self,"pre-delitem")) return -1;
v = (PyObject *)object_from_oid(self,key);
if(!v) return -1;
if(PyExtensionClass_Check(v))
{
self->klass_count--;
}
else
{
if(((cPersistentObject*)v)->state>=0)
{
self->non_ghost_count--;
((cPersistentObject*)v)->ring.next->prev = ((cPersistentObject*)v)->ring.prev;
((cPersistentObject*)v)->ring.prev->next = ((cPersistentObject*)v)->ring.next;
((cPersistentObject*)v)->ring.prev = NULL;
((cPersistentObject*)v)->ring.next = NULL;
}
else
{
/* This is a ghost object, so we havent kept a reference count on it.
For it have stayed alive this long someone else must be keeping a reference
to it. Therefore we need to temporarily give it back a reference count
before calling DelItem below */
Py_INCREF(v);
}
Py_DECREF(((cPersistentObject*)v)->cache);
((cPersistentObject*)v)->cache = NULL;
}
Py_DECREF(v);
if(PyDict_DelItem(self->data, key))
{
PyErr_SetString(PyExc_RuntimeError,
"unexpectedly couldnt remove key in cc_ass_sub");
return -1;
}
if(check_ring(self,"post-delitem")) return -1;
return 0;
} }
return PyDict_DelItem(self->data, key);
} }
static int _check_ring(ccobject *self,const char *context)
{
CPersistentRing *here = &(self->ring_home);
int expected = 1+self->non_ghost_count;
int total = 0;
do
{
if(++total>(expected+10)) return 3; /* ring too big, by a large margin */
if(!here->next) return 4; /* various linking problems */
if(!here->prev) return 5;
if(!here->next->prev) return 7;
if(!here->prev->next) return 8;
if(here->prev->next!=here) return 9;
if(here->next->prev!=here) return 10;
if(!self->ring_lock)
{
/* if the ring must be locked then it only contains object other than persistent instances */
if(here!=&self->ring_home)
{
cPersistentObject *object = object_from_ring(self,here,context);
if(!object) return 12;
if(object->state==cPersistent_GHOST_STATE)
return 13;
}
}
here = here->next;
}
while(here!=&self->ring_home);
if(self->ring_lock)
{
if(total<expected) return 6; /* ring too small; too big is ok when locked */
}
else
{
if(total!=expected) return 14; /* ring size wrong, or bad ghost accounting */
}
return 0;
}
static int check_ring(ccobject *self,const char *context)
{
#ifdef MUCH_RING_CHECKING
int code=_check_ring(self,context);
if(code)
{
/*printf(stderr,"BROKEN RING (code %d) in %s, size %d\n",code,context,PyDict_Size(self->data));*/
PyErr_Format(PyExc_RuntimeError,"broken ring (code %d) in %s, size %d",code,context,PyDict_Size(self->data));
return code;
}
#endif
return 0;
}
static int
present_in_ring(ccobject *self,CPersistentRing *target)
{
CPersistentRing *here = self->ring_home.next;
while(1)
{
if(here==target)
{
return 1;
}
if(here==&self->ring_home)
{
/* back to the home position, and we didnt find it */
return 0;
}
here = here->next;
}
}
static PyMappingMethods cc_as_mapping = { static PyMappingMethods cc_as_mapping = {
(inquiry)cc_length, /*mp_length*/ (inquiry)cc_length, /*mp_length*/
(binaryfunc)cc_subscript, /*mp_subscript*/ (binaryfunc)cc_subscript, /*mp_subscript*/
...@@ -636,6 +912,10 @@ static PyTypeObject Cctype = { ...@@ -636,6 +912,10 @@ static PyTypeObject Cctype = {
(hashfunc)0, /*tp_hash*/ (hashfunc)0, /*tp_hash*/
(ternaryfunc)0, /*tp_call*/ (ternaryfunc)0, /*tp_call*/
(reprfunc)0, /*tp_str*/ (reprfunc)0, /*tp_str*/
/* Space for future expansion */
0L,0L,0L,0L,
""
}; };
static PyObject * static PyObject *
...@@ -644,9 +924,9 @@ cCM_new(PyObject *self, PyObject *args) ...@@ -644,9 +924,9 @@ cCM_new(PyObject *self, PyObject *args)
int cache_size=100, cache_age=1000; int cache_size=100, cache_age=1000;
PyObject *jar; PyObject *jar;
if (!PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age)) UNLESS(PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age))
return NULL; return NULL;
return (PyObject *)newccobject(jar, cache_size, cache_age); return (PyObject*)newccobject(jar, cache_size,cache_age);
} }
static struct PyMethodDef cCM_methods[] = { static struct PyMethodDef cCM_methods[] = {
...@@ -657,12 +937,11 @@ static struct PyMethodDef cCM_methods[] = { ...@@ -657,12 +937,11 @@ static struct PyMethodDef cCM_methods[] = {
void void
initcPickleCache(void) initcPickleCache(void)
{ {
PyObject *m; PyObject *m, *d;
Cctype.ob_type = &PyType_Type; Cctype.ob_type=&PyType_Type;
if (!ExtensionClassImported) UNLESS(ExtensionClassImported) return;
return;
m = Py_InitModule4("cPickleCache", cCM_methods, cPickleCache_doc_string, m = Py_InitModule4("cPickleCache", cCM_methods, cPickleCache_doc_string,
(PyObject*)NULL, PYTHON_API_VERSION); (PyObject*)NULL, PYTHON_API_VERSION);
...@@ -670,4 +949,15 @@ initcPickleCache(void) ...@@ -670,4 +949,15 @@ initcPickleCache(void)
py_reload = PyString_InternFromString("reload"); py_reload = PyString_InternFromString("reload");
py__p_jar = PyString_InternFromString("_p_jar"); py__p_jar = PyString_InternFromString("_p_jar");
py__p_changed = PyString_InternFromString("_p_changed"); py__p_changed = PyString_InternFromString("_p_changed");
py__p_oid = PyString_InternFromString("_p_oid");
d = PyModule_GetDict(m);
PyDict_SetItemString(d,"cache_variant",PyString_FromString("stiff/c"));
#ifdef MUCH_RING_CHECKING
PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(1));
#else
PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(0));
#endif
} }
src/persistent/cPersistence.c
View file @ 226ee421
...@@ -14,11 +14,21 @@ ...@@ -14,11 +14,21 @@
static char cPersistence_doc_string[] = static char cPersistence_doc_string[] =
"Defines Persistent mixin class for persistent objects.\n" "Defines Persistent mixin class for persistent objects.\n"
"\n" "\n"
"$Id: cPersistence.c,v 1.50 2002/03/08 18:36:13 jeremy Exp $\n"; "$Id: cPersistence.c,v 1.51 2002/03/27 10:14:04 htrd Exp $\n";
#include <string.h> #include <string.h>
#include "cPersistence.h" #include "cPersistence.h"
/* the layout of this struct is the same as the start of ccobject in cPickleCache.c */
struct ccobject_head_struct {
PyObject_HEAD
CPersistentRing ring_home;
int non_ghost_count;
};
#define HOME(O) ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->ring_home)) )
#define NON_GHOST_COUNT(O) ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->non_ghost_count)) )
#define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;} #define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
#define UNLESS(E) if(!(E)) #define UNLESS(E) if(!(E))
#define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V) #define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
...@@ -112,21 +122,82 @@ if(self->state < 0 && self->jar) \ ...@@ -112,21 +122,82 @@ if(self->state < 0 && self->jar) \
{ \ { \
PyObject *r; \ PyObject *r; \
\ \
int *count = NON_GHOST_COUNT(self); \
if(count) \
{ \
(*count)++; \
self->ring.next = HOME(self); \
self->ring.prev = HOME(self)->prev; \
HOME(self)->prev->next = &self->ring; \
HOME(self)->prev = &self->ring; \
Py_INCREF(self); \
} \
self->state=cPersistent_CHANGED_STATE; \ self->state=cPersistent_CHANGED_STATE; \
UNLESS(r=callmethod1(self->jar,py_setstate,(PyObject*)self)) \ UNLESS(r=callmethod1(self->jar,py_setstate,(PyObject*)self)) \
{ \ { \
self->state=cPersistent_GHOST_STATE; \ ghostify(self); \
return ER; \ return ER; \
} \ } \
self->state=cPersistent_UPTODATE_STATE; \ self->state=cPersistent_UPTODATE_STATE; \
Py_DECREF(r); \ Py_DECREF(r); \
} }
#define KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self) \
if(HOME(self) && self->state>=0) { \
self->ring.prev->next = self->ring.next; \
self->ring.next->prev = self->ring.prev; \
self->ring.next = HOME(self); \
self->ring.prev = HOME(self)->prev; \
HOME(self)->prev->next = &self->ring; \
HOME(self)->prev = &self->ring; }
/****************************************************************************/ /****************************************************************************/
staticforward PyExtensionClass Pertype; staticforward PyExtensionClass Pertype;
static void
accessed(cPersistentObject *self)
{
KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
}
static void
ghostify(cPersistentObject *self)
{
int *count;
count = NON_GHOST_COUNT(self);
if(count && (self->state>=0))
{
(*count)--;
self->ring.next->prev = self->ring.prev;
self->ring.prev->next = self->ring.next;
self->ring.prev = NULL;
self->ring.next = NULL;
self->state = cPersistent_GHOST_STATE;
Py_DECREF(self);
}
else
{
self->state = cPersistent_GHOST_STATE;
}
}
static void
deallocated(cPersistentObject *self)
{
if(self->state>=0) ghostify(self);
if(self->cache)
{
PyObject *v=PyObject_CallMethod(self->cache,"_oid_unreferenced","O",self->oid);
if(!v) PyErr_Clear(); /* and explode later */
Py_XDECREF(v);
}
Py_XDECREF(self->jar);
Py_XDECREF(self->oid);
}
static int static int
changed(cPersistentObject *self) changed(cPersistentObject *self)
{ {
...@@ -185,7 +256,7 @@ Per___changed__(cPersistentObject *self, PyObject *args) ...@@ -185,7 +256,7 @@ Per___changed__(cPersistentObject *self, PyObject *args)
static PyObject * static PyObject *
Per__p_deactivate(cPersistentObject *self, PyObject *args) Per__p_deactivate(cPersistentObject *self, PyObject *args)
{ {
PyObject *dict; PyObject *dict,*dict2=NULL;
#ifdef DEBUG_LOG #ifdef DEBUG_LOG
if (idebug_log < 0) call_debug("reinit",self); if (idebug_log < 0) call_debug("reinit",self);
...@@ -197,13 +268,22 @@ Per__p_deactivate(cPersistentObject *self, PyObject *args) ...@@ -197,13 +268,22 @@ Per__p_deactivate(cPersistentObject *self, PyObject *args)
if (self->state==cPersistent_UPTODATE_STATE && self->jar && if (self->state==cPersistent_UPTODATE_STATE && self->jar &&
HasInstDict(self) && (dict=INSTANCE_DICT(self))) HasInstDict(self) && (dict=INSTANCE_DICT(self)))
{ {
dict2 = PyDict_Copy(dict);
PyDict_Clear(dict); PyDict_Clear(dict);
/* Note that we need to set to ghost state unless we are /* Note that we need to set to ghost state unless we are
called directly. Methods that override this need to called directly. Methods that override this need to
do the same! */ do the same! */
self->state=cPersistent_GHOST_STATE; ghostify(self);
} }
/* need to delay releasing the last reference on instance attributes
until after we have finished accounting for losing our state */
if(dict2)
{
PyDict_Clear(dict2);
Py_DECREF(dict2);
}
Py_INCREF(Py_None); Py_INCREF(Py_None);
return Py_None; return Py_None;
} }
...@@ -333,8 +413,8 @@ Per_dealloc(cPersistentObject *self) ...@@ -333,8 +413,8 @@ Per_dealloc(cPersistentObject *self)
#ifdef DEBUG_LOG #ifdef DEBUG_LOG
if(idebug_log < 0) call_debug("del",self); if(idebug_log < 0) call_debug("del",self);
#endif #endif
Py_XDECREF(self->jar); deallocated(self);
Py_XDECREF(self->oid); Py_XDECREF(self->cache);
Py_DECREF(self->ob_type); Py_DECREF(self->ob_type);
PyObject_DEL(self); PyObject_DEL(self);
} }
...@@ -387,7 +467,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name, ...@@ -387,7 +467,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
{ {
UPDATE_STATE_IF_NECESSARY(self, NULL); UPDATE_STATE_IF_NECESSARY(self, NULL);
self->atime=((long)(time(NULL)/3))%65536; KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
if (self->serial[7]=='\0' && self->serial[6]=='\0' && if (self->serial[7]=='\0' && self->serial[6]=='\0' &&
self->serial[5]=='\0' && self->serial[4]=='\0' && self->serial[5]=='\0' && self->serial[4]=='\0' &&
...@@ -419,7 +499,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name, ...@@ -419,7 +499,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
{ {
UPDATE_STATE_IF_NECESSARY(self, NULL); UPDATE_STATE_IF_NECESSARY(self, NULL);
self->atime=((long)(time(NULL)/3))%65536; KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
} }
return getattrf((PyObject *)self, oname); return getattrf((PyObject *)self, oname);
...@@ -466,6 +546,21 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v, ...@@ -466,6 +546,21 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
{ {
if(name[3]=='o' && name[4]=='i' && name[5]=='d' && ! name[6]) if(name[3]=='o' && name[4]=='i' && name[5]=='d' && ! name[6])
{ {
if(HOME(self))
{
int result;
if(!v)
{
PyErr_SetString(PyExc_ValueError,"can not delete the oid of a cached object");
return -1;
}
if(PyObject_Cmp(self->oid,v,&result)<0) return -1;
if(result)
{
PyErr_SetString(PyExc_ValueError,"can not change the oid of a cached object");
return -1;
}
}
Py_XINCREF(v); Py_XINCREF(v);
ASSIGN(self->oid, v); ASSIGN(self->oid, v);
return 0; return 0;
...@@ -509,7 +604,6 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v, ...@@ -509,7 +604,6 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
v=PyObject_GetAttr(OBJECT(self), py__p_deactivate); v=PyObject_GetAttr(OBJECT(self), py__p_deactivate);
if (v) { ASSIGN(v, PyObject_CallObject(v, NULL)); } if (v) { ASSIGN(v, PyObject_CallObject(v, NULL)); }
if (v) { Py_DECREF(v); } if (v) { Py_DECREF(v); }
self->state=cPersistent_GHOST_STATE;
return 0; return 0;
} }
if (PyObject_IsTrue(v)) return changed(self); if (PyObject_IsTrue(v)) return changed(self);
...@@ -521,8 +615,7 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v, ...@@ -521,8 +615,7 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
{ {
UPDATE_STATE_IF_NECESSARY(self, -1); UPDATE_STATE_IF_NECESSARY(self, -1);
/* Record access times */ KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
self->atime=((long)(time(NULL)/3))%65536;
if((! (*name=='_' && name[1]=='v' && name[2]=='_')) if((! (*name=='_' && name[1]=='v' && name[2]=='_'))
&& (self->state != cPersistent_CHANGED_STATE && self->jar) && (self->state != cPersistent_CHANGED_STATE && self->jar)
...@@ -680,9 +773,11 @@ truecPersistenceCAPI = { ...@@ -680,9 +773,11 @@ truecPersistenceCAPI = {
(getattrofunc)Per_getattro, /*tp_getattr with object key*/ (getattrofunc)Per_getattro, /*tp_getattr with object key*/
(setattrofunc)Per_setattro, /*tp_setattr with object key*/ (setattrofunc)Per_setattro, /*tp_setattr with object key*/
changed, changed,
accessed,
ghostify,
deallocated,
(intfunctionwithpythonarg)Per_setstate, (intfunctionwithpythonarg)Per_setstate,
(pergetattr)Per_getattr, (pergetattr)Per_getattr,
(persetattr)_setattro,
}; };
void void
......
src/persistent/cPersistence.h
View file @ 226ee421
...@@ -18,12 +18,21 @@ ...@@ -18,12 +18,21 @@
#include "ExtensionClass.h" #include "ExtensionClass.h"
#include <time.h> #include <time.h>
#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid; char serial[8]; unsigned short atime; signed char state; unsigned char reserved;
#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid, *cache; CPersistentRing ring; char serial[8]; signed char state; unsigned char reserved[3];
#define cPersistent_GHOST_STATE -1 #define cPersistent_GHOST_STATE -1
#define cPersistent_UPTODATE_STATE 0 #define cPersistent_UPTODATE_STATE 0
#define cPersistent_CHANGED_STATE 1 #define cPersistent_CHANGED_STATE 1
#define cPersistent_STICKY_STATE 2 #define cPersistent_STICKY_STATE 2
struct ccobject_head_struct;
typedef struct CPersistentRing_struct
{
struct CPersistentRing_struct *prev;
struct CPersistentRing_struct *next;
} CPersistentRing;
typedef struct { typedef struct {
cPersistent_HEAD cPersistent_HEAD
} cPersistentObject; } cPersistentObject;
...@@ -36,6 +45,9 @@ typedef struct { ...@@ -36,6 +45,9 @@ typedef struct {
getattrofunc getattro; getattrofunc getattro;
setattrofunc setattro; setattrofunc setattro;
int (*changed)(cPersistentObject*); int (*changed)(cPersistentObject*);
void (*accessed)(cPersistentObject*);
void (*ghostify)(cPersistentObject*);
void (*deallocated)(cPersistentObject*);
int (*setstate)(PyObject*); int (*setstate)(PyObject*);
pergetattr pergetattro; pergetattr pergetattro;
persetattr persetattro; persetattr persetattro;
...@@ -59,11 +71,13 @@ static cPersistenceCAPIstruct *cPersistenceCAPI; ...@@ -59,11 +71,13 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
#define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O))) #define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O)))
#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE)) #define PER_GHOSTIFY(O) (cPersistenceCAPI->ghostify((cPersistentObject*)(O)))
#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
#define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE)) #define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
#define PER_DEL(O) Py_XDECREF((O)->jar); Py_XDECREF((O)->oid); #define PER_DEL(O) (cPersistenceCAPI->deallocated((cPersistentObject*)(O)))
#define PER_USE(O) \ #define PER_USE(O) \
(((O)->state != cPersistent_GHOST_STATE \ (((O)->state != cPersistent_GHOST_STATE \
...@@ -71,7 +85,7 @@ static cPersistenceCAPIstruct *cPersistenceCAPI; ...@@ -71,7 +85,7 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
? (((O)->state==cPersistent_UPTODATE_STATE) \ ? (((O)->state==cPersistent_UPTODATE_STATE) \
? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0) ? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0)
#define PER_ACCESSED(O) ((O)->atime=((long)(time(NULL)/3))%65536) #define PER_ACCESSED(O) (cPersistenceCAPI->accessed((cPersistentObject*)(O)))
#endif #endif
......
src/persistent/cPickleCache.c
View file @ 226ee421
...@@ -2,448 +2,531 @@ ...@@ -2,448 +2,531 @@
Copyright (c) 2001, 2002 Zope Corporation and Contributors. Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved. All Rights Reserved.
This software is subject to the provisions of the Zope Public License, This software is subject to the provisions of the Zope Public License,
Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE FOR A PARTICULAR PURPOSE
****************************************************************************/ ****************************************************************************/
static char cPickleCache_doc_string[] =
static char cPickleCache_doc_string[] =
"Defines the PickleCache used by ZODB Connection objects.\n" "Defines the PickleCache used by ZODB Connection objects.\n"
"\n" "\n"
"$Id: cPickleCache.c,v 1.41 2002/03/08 18:36:14 jeremy Exp $\n"; "$Id: cPickleCache.c,v 1.42 2002/03/27 10:14:04 htrd Exp $\n";
/* Compute the current time in the units and range used for peristent #define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
objects. */ #define UNLESS(E) if(!(E))
#define PER_TIME() ((long)(time(NULL) / 3)) % 65536 #define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
#define OBJECT(O) ((PyObject*)O)
#define DONT_USE_CPERSISTENCECAPI #define DONT_USE_CPERSISTENCECAPI
#include "cPersistence.h" #include "cPersistence.h"
#include <time.h> #include <time.h>
#include <stddef.h>
#undef Py_FindMethod #undef Py_FindMethod
static PyObject *py_reload, *py__p_jar, *py__p_changed;
typedef struct { static PyObject *py__p_oid, *py_reload, *py__p_jar, *py__p_changed;
PyObject_HEAD
PyObject *data;
PyObject *jar;
PyObject *setklassstate;
int position;
int cache_size;
int cache_age;
/* Cache statistics */
int sum_deal;
int sum_deac;
double sum_age;
int n, na;
time_t last_check; /* Time of last gc */
double mean_age;
double mean_deal;
double mean_deac;
double df, dfa; /* Degees of freedom for above stats */
} ccobject;
#define WEIGHTING_PERIOD 600 /* define this for extra debugging checks, and lousy performance */
#define MUCH_RING_CHECKING 1
/* /* Do we want 'engine noise'.... abstract debugging output useful for
How to compute weighted means? visualizing cache behavior */
#if 0
#define ENGINE_NOISE(A) printf(A)
#else
#define ENGINE_NOISE(A) ((void)A)
#endif
Assume we have two means, a current mean, M, and a mean as of some /* the layout of this struct is the same as the start of ccobject_head in cPersistence.c */
time d seconds in the past, Md. The means have effective degrees typedef struct {
of freedom, N, and Nd. Where Nd is adjusted by d is some fashion. PyObject_HEAD
The combined mean is (M*N+Md*Nd)/(N+Nd). The degrees of freedom CPersistentRing ring_home;
of the combined mean, Nc, is N+Nd. Nd is computed by weighting int non_ghost_count;
an old degree of freedom with the weight: I/(I+d), where I is some int klass_count;
suitably chosen constant, which we will call a "weighting period". PyObject *data;
PyObject *jar;
*/ PyObject *setklassstate;
int cache_size;
int ring_lock;
int cache_drain_resistance;
} ccobject;
staticforward PyTypeObject Cctype; staticforward PyTypeObject Cctype;
staticforward int present_in_ring(ccobject *self,CPersistentRing *target);
staticforward int check_ring(ccobject *self,const char *context);
staticforward int cc_ass_sub(ccobject *self, PyObject *key, PyObject *v);
/* ---------------------------------------------------------------- */ /* ---------------------------------------------------------------- */
static int static PyObject *object_from_oid(ccobject *self,PyObject *key)
gc_item(ccobject *self, PyObject *key, PyObject *v, long now, int dt) /* somewhat of a replacement for PyDict_GetItem(self->data....
however this returns a *new* reference */
{ {
if (!(v && key)) PyObject *v = PyDict_GetItem(self->data, key);
return 0; if(!v) return NULL;
self->n++;
/* If there is at most one reference to this object, then the
cache has the only reference. It can be removed. */
if (v->ob_refcnt <= 1) {
self->sum_deal++;
/* XXX The fact that this works will iterating over
self->data with PyDict_Next() is an accident of the
current Python dictionary implementation. */
return PyDict_DelItem(self->data, key);
}
if (dt >= 0 && Py_INCREF(v);
(!PyExtensionClass_Check(v)) &&
((cPersistentObject*)v)->jar == self->jar /* I'm paranoid */ && return v;
((cPersistentObject*)v)->state == cPersistent_UPTODATE_STATE) {
now -= ((cPersistentObject*)v)->atime;
if (now < 0)
now += 65536;
self->na++;
self->sum_age += now;
if (now > dt) {
/* We have a cPersistent object that hasn't been used in
a while. Reinitialize it, hopefully freeing it's
state.
*/
self->sum_deac++;
if (PyObject_SetAttr(v, py__p_changed, Py_None) < 0)
PyErr_Clear();
}
}
return 0;
} }
static void static cPersistentObject *object_from_ring(ccobject *self,CPersistentRing *here,const char *context)
update_stats(ccobject *self, time_t now)
{ {
double d, deal, deac; /* Given a position in the LRU ring, return a borrowed
reference to the object at that point in the ring. The caller is
d = now - self->last_check; responsible for ensuring that this ring position really does
if(d < 1) correspond to a persistent object, although the debugging
return; version will double-check this. */
self->df *= WEIGHTING_PERIOD / (WEIGHTING_PERIOD + d); PyObject *object = (PyObject *)(((char *)here)-offsetof(cPersistentObject,ring));
self->dfa *= WEIGHTING_PERIOD / (WEIGHTING_PERIOD + d);
#ifdef MUCH_RING_CHECKING
self->mean_age = ((self->mean_age * self->dfa + self->sum_age)/ if(!PyExtensionInstance_Check(object))
(self->dfa + self->na)) * 3; {
self->sum_age = 0; PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered non-ExtensionClass object in %s",context);
return NULL;
deac = self->sum_deac / d; }
self->sum_deac = 0; if(!(((PyExtensionClass*)(object->ob_type))->class_flags & PERSISTENT_TYPE_FLAG))
self->mean_deac = ((self->mean_deac * self->dfa+deac)/ {
(self->dfa + self->na)); PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered non-persistent object in %s",context);
self->sum_deac = 0; return NULL;
}
self->dfa += self->na; if(((cPersistentObject*)object)->jar!=self->jar)
self->na = 0; {
PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered object from a different jar in %s",context);
deal=self->sum_deal/d; return NULL;
self->sum_deal = 0; }
self->mean_deal = ((self->mean_deal * self->df + deal)/ if(((cPersistentObject*)object)->cache!=(PyObject *)self)
(self->df +self->n)); {
self->sum_deal = 0; PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered broken ring in %s",context);
return NULL;
self->df += self->n; }
self->n = 0; #endif
return (cPersistentObject *)object;
self->last_check = now;
} }
static int static int
check_size(ccobject *self) scan_gc_items(ccobject *self,int target)
{ {
if (self->cache_size < 1) cPersistentObject *object;
return 0; int error;
return PyDict_Size(self->data); CPersistentRing placeholder;
CPersistentRing *here = self->ring_home.next;
#ifdef MUCH_RING_CHECKING
int safety_counter = self->cache_size*10;
if(safety_counter<10000) safety_counter = 10000;
#endif
while(1)
{
if(check_ring(self,"mid-gc")) return -1;
#ifdef MUCH_RING_CHECKING
if(!safety_counter--)
{
/* This loop has been running for a very long time.
It is possible that someone loaded a very large number of objects,
and now wants us to blow them all away. However it may
also indicate a logic error. If the loop has been running this
long then you really have to doubt it will ever terminate.
In the MUCH_RING_CHECKING build we prefer to raise an exception
here */
PyErr_SetString(PyExc_RuntimeError,"scan_gc_items safety counter exceeded");
return -1;
}
if(!present_in_ring(self,here))
{
/* Our current working position is no longer in the ring. Thats bad. */
PyErr_SetString(PyExc_RuntimeError,"working position fell out the ring, in scan_gc_items");
return -1;
}
#endif
if(here==&self->ring_home)
{
/* back to the home position. stop looking */
return 0;
}
/* At this point we know that the ring only contains nodes from
persistent objects, plus our own home node. We can safely
assume this is a persistent object now we know it is not the home */
object = object_from_ring(self,here,"scan_gc_items");
if(!object) return -1;
if(self->non_ghost_count<=target)
{
/* we are small enough */
return 0;
}
else if(object->state==cPersistent_UPTODATE_STATE)
{
/* deactivate it. This is the main memory saver. */
ENGINE_NOISE("G");
/* add a placeholder */
placeholder.next = here->next;
placeholder.prev = here;
here->next->prev = &placeholder;
here->next = &placeholder;
error = PyObject_SetAttr((PyObject *)object,py__p_changed,Py_None);
/* unlink the placeholder */
placeholder.next->prev=placeholder.prev;
placeholder.prev->next=placeholder.next;
here = placeholder.next;
if(error)
return -1; /* problem */
}
else
{
ENGINE_NOISE(".");
here = here->next;
}
}
} }
static int
gc_all_items(ccobject *self, int now, int dt) static PyObject *
lockgc(ccobject *self,int target_size)
{ {
PyObject *key, *v; if(self->ring_lock)
int i; {
Py_INCREF(Py_None);
return Py_None;
}
for(i = 0; PyDict_Next(self->data, &i, &key, &v); ) if(check_ring(self,"pre-gc")) return NULL;
if (gc_item(self, key, v, now, dt) < 0) ENGINE_NOISE("<");
return -1; self->ring_lock = 1;
return 0; if(scan_gc_items(self,target_size))
{
self->ring_lock = 0;
return NULL;
}
self->ring_lock = 0;
ENGINE_NOISE(">\n");
if(check_ring(self,"post-gc")) return NULL;
Py_INCREF(Py_None);
return Py_None;
} }
static int static PyObject *
fullgc(ccobject *self, int dt) cc_incrgc(ccobject *self, PyObject *args)
{ {
long now; int n=1;
if (check_size(self) <= 0) int starting_size = self->non_ghost_count;
return 0;
now = PER_TIME(); int target_size = self->cache_size;
dt /= 3;
if (gc_all_items(self, now, dt) < 0) if(self->cache_drain_resistance>=1)
return -1; {
self->position = 0; /* This cache will gradually drain down to a small size. Check
a (small) number of objects proportional to the current size */
if (now - self->last_check > 1) int target_size_2 = starting_size - 1 - starting_size/self->cache_drain_resistance;
update_stats(self, now); if(target_size_2<target_size)
target_size = target_size_2;
return 0;
}
static int
reallyfullgc(ccobject *self, int dt)
{
int l, last;
time_t now;
last = check_size(self);
if (last <= 0)
return 0;
now = PER_TIME();
/* Units are 3 seconds */
dt /= 3;
/* First time through should get refcounts to 1 */
if (gc_all_items(self, now, dt) < 0)
return -1;
l = PyDict_Size(self->data);
if (l < 0)
return -1;
/* Now continue to collect until the size of the cache stops
decreasing. */
while (l < last) {
if (gc_all_items(self, now, dt) < 0)
return -1;
last = l;
l = PyDict_Size(self->data);
if (l < 0)
return -1;
} }
if (now - self->last_check > 1)
update_stats(self, now);
self->position = 0;
return 0;
}
static int UNLESS (PyArg_ParseTuple(args, "|i",&n)) return NULL;
maybegc(ccobject *self, PyObject *thisv)
{
int n, s, size, dt;
long now;
PyObject *key=0, *v=0;
s = check_size(self);
if (s <= 0)
return 0;
now = PER_TIME();
size = self->cache_size;
self->cache_size = 0;
/* Decide how many objects to look at */
n = (s - size) / 10;
if (n < 3)
n = 3;
/* Decide how much time to give them before deactivating them */
s = 8 * size / s;
if (s > 100)
s = 100;
dt = (long)(self->cache_age * (0.2 + 0.1 * s));
/* Units are 3 seconds */
dt /= 3;
while (--n >= 0) {
if (PyDict_Next(self->data, &(self->position), &key, &v)) {
if (v != thisv && gc_item(self, key, v, now, dt) < 0) {
self->cache_size=size;
return -1;
}
}
else
self->position = 0;
}
self->cache_size = size;
if (now - self->last_check > 1)
update_stats(self, now);
return 0; return lockgc(self,target_size);
} }
static PyObject * static PyObject *
cc_full_sweep(ccobject *self, PyObject *args) cc_full_sweep(ccobject *self, PyObject *args)
{ {
int dt = self->cache_age; int dt=0;
if (!PyArg_ParseTuple(args, "|i:full_sweep", &dt)) UNLESS(PyArg_ParseTuple(args, "|i", &dt)) return NULL;
return NULL; return lockgc(self,0);
if (dt < -1) {
PyErr_SetString(PyExc_ValueError, "age must be >= -1");
return NULL;
}
if (fullgc(self, dt) == -1)
return NULL;
Py_INCREF(Py_None);
return Py_None;
} }
static PyObject * static PyObject *
cc_reallyfull_sweep(ccobject *self, PyObject *args) cc_reallyfull_sweep(ccobject *self, PyObject *args)
{ {
int dt = self->cache_age; int dt=0;
if (!PyArg_ParseTuple(args, "|i:minimize", &dt)) UNLESS(PyArg_ParseTuple(args, "|i", &dt)) return NULL;
return NULL; return lockgc(self,0);
if (dt < -1) {
PyErr_SetString(PyExc_ValueError, "age must be >= -1");
return NULL;
}
if (reallyfullgc(self, dt) == -1)
return NULL;
Py_INCREF(Py_None);
return Py_None;
} }
static PyObject * static void
cc_incrgc(ccobject *self, PyObject *args) _invalidate(ccobject *self, PyObject *key)
{ {
int n = 1; PyObject *v=object_from_oid(self, key);
if (!PyArg_ParseTuple(args, "|i:incrgr", &n)) if(!v)
return NULL; {
/* shouldnt this be an error? for now Ill follow Jims lead */
PyErr_Clear();
}
else
{
if (PyExtensionClass_Check(v))
{
if(v->ob_refcnt <= 1)
{
self->klass_count--;
if (PyDict_DelItem(self->data, key) < 0)
PyErr_Clear();
}
else
{
v=PyObject_CallFunction(self->setklassstate,
"O", v);
if (v) Py_DECREF(v);
else PyErr_Clear();
}
}
else
{
if(PyObject_DelAttr(v,py__p_changed) < 0)
PyErr_Clear();
}
Py_DECREF(v);
}
}
for (; --n >= 0;) static PyObject *
if (maybegc(self, NULL) < 0) cc_invalidate(ccobject *self, PyObject *args)
return NULL; {
PyObject *inv, *key, *v;
int i;
if (PyArg_ParseTuple(args, "O!", &PyDict_Type, &inv)) {
for (i=0; PyDict_Next(inv, &i, &key, &v); )
if (key==Py_None)
{ /* Eek some nitwit invalidated everything! */
for (i=0; PyDict_Next(self->data, &i, &key, &v); )
_invalidate(self, key);
break;
}
else
_invalidate(self, key);
PyDict_Clear(inv);
}
else {
PyErr_Clear();
UNLESS (PyArg_ParseTuple(args, "O", &inv)) return NULL;
if (PyString_Check(inv))
_invalidate(self, inv);
else if (inv==Py_None) /* All */
for (i=0; PyDict_Next(self->data, &i, &key, &v); )
_invalidate(self, key);
else {
int l;
PyErr_Clear();
if ((l=PyObject_Length(inv)) < 0) return NULL;
for(i=l; --i >= 0; )
{
UNLESS (key=PySequence_GetItem(inv, i)) return NULL;
_invalidate(self, key);
Py_DECREF(key);
}
PySequence_DelSlice(inv, 0, l);
}
}
Py_INCREF(Py_None); Py_INCREF(Py_None);
return Py_None; return Py_None;
} }
static void
_invalidate(ccobject *self, PyObject *key) static PyObject *
cc_get(ccobject *self, PyObject *args)
{ {
PyObject *v = PyDict_GetItem(self->data, key); PyObject *r, *key, *d=0;
if (!v) UNLESS (PyArg_ParseTuple(args,"O|O", &key, &d)) return NULL;
return;
if (PyExtensionClass_Check(v)) UNLESS (r=(PyObject *)object_from_oid(self, key))
if (v->ob_refcnt <= 1) { {
self->sum_deal++; if (d)
if (PyDict_DelItem(self->data, key) < 0) {
PyErr_Clear(); PyErr_Clear();
} else { r=d;
PyObject *t = PyTuple_New(1); Py_INCREF(r);
if (t) { }
PyTuple_SET_ITEM(t, 0, v); else
v = PyObject_CallObject(self->setklassstate, t); {
/* Set tuple element to NULL so that deallocating the PyErr_SetObject(PyExc_KeyError, key);
tuple does not decref t. return NULL;
*/
PyTuple_SET_ITEM(t, 0, NULL);
Py_DECREF(t);
} else
v = t;
if (v)
Py_DECREF(v);
else
PyErr_Clear();
} }
else if (PyObject_DelAttr(v, py__p_changed) < 0) }
PyErr_Clear();
return r;
} }
static void static PyObject *
_invalidate_all(ccobject *self) cc_klass_items(ccobject *self, PyObject *args)
{ {
PyObject *key, *v; PyObject *l,*k,*v;
int i; int p = 0;
if(!PyArg_ParseTuple(args,"")) return NULL;
l = PyList_New(0);
if(!l) return NULL;
while(PyDict_Next(self->data, &p, &k, &v))
{
if(PyExtensionClass_Check(v))
{
v=PyObject_CallMethod(l,"append","((OO))",k,v);
if(!v)
{
Py_DECREF(l);
return NULL;
}
}
}
for (i = 0; PyDict_Next(self->data, &i, &key, &v); ) return l;
_invalidate(self, key);
} }
static PyObject * static PyObject *
cc_invalidate(ccobject *self, PyObject *args) cc_lru_items(ccobject *self, PyObject *args)
{ {
PyObject *inv, *key, *v; PyObject *l;
int i; CPersistentRing *here;
if (!PyArg_ParseTuple(args, "O:invalidate", &inv)) if(!PyArg_ParseTuple(args,"")) return NULL;
return NULL;
if (PyDict_Check(inv)) { if(self->ring_lock)
for (i = 0; PyDict_Next(inv, &i, &key, &v); ) {
if (key == Py_None) { PyErr_SetString(PyExc_ValueError,".lru_items() is unavailable during garbage collection");
/* Eek some nitwit invalidated everything! */ return NULL;
_invalidate_all(self);
break;
}
else
_invalidate(self, key);
PyDict_Clear(inv);
} else if (PyString_Check(inv))
_invalidate(self, inv);
else if (inv == Py_None) /* All */
_invalidate_all(self);
else {
int l = PyObject_Length(inv);
if (l < 0)
return NULL;
for (i = l; --i >= 0; ) {
key = PySequence_GetItem(inv, i);
if (!key)
return NULL;
_invalidate(self, key);
Py_DECREF(key);
}
PySequence_DelSlice(inv, 0, l);
} }
Py_INCREF(Py_None); if(check_ring(self,"pre-cc_items")) return NULL;
return Py_None;
l = PyList_New(0);
if(!l) return NULL;
here = self->ring_home.next;
while(here!=&self->ring_home)
{
cPersistentObject *object = object_from_ring(self,here,"cc_items");
PyObject *v;
if(!object)
{
Py_DECREF(l);
return NULL;
}
v=PyObject_CallMethod(l,"append","((OO))",object->oid,object);
if(!v)
{
Py_DECREF(l);
return NULL;
}
Py_DECREF(v);
here = here->next;
}
return l;
} }
static PyObject * static PyObject *
cc_get(ccobject *self, PyObject *args) cc_oid_unreferenced(ccobject *self, PyObject *args)
{ {
PyObject *r, *key, *d = NULL; PyObject *oid,*v;
if(!PyArg_ParseTuple(args,"O",&oid)) return NULL;
if (!PyArg_ParseTuple(args, "O|O:get", &key, &d)) v = PyDict_GetItem(self->data, oid);
return NULL; if(!v) return NULL;
r = PyDict_GetItem(self->data, key); if(v->ob_refcnt)
if (!r) { {
if (d) PyErr_Format(PyExc_ValueError,"object has reference count of %d, should be zero",v->ob_refcnt);
r = d; return NULL;
else {
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
}
} }
Py_INCREF(r); /* Need to be very hairy here because a dictionary is about
return r; to decref an already deleted object */
#ifdef Py_TRACE_REFS
#error "this code path has not been tested - Toby Dickenson"
_Py_NewReference(v);
/* it may be a problem that v->ob_type is still NULL? */
#else
Py_INCREF(v);
#endif
if(v->ob_refcnt!=1)
{
PyErr_SetString(PyExc_ValueError,"refcount is not 1 after resurrection");
return NULL;
}
/* return the stolen reference */
Py_INCREF(v);
PyDict_DelItem(self->data, oid);
if(v->ob_refcnt!=1)
{
PyErr_SetString(PyExc_ValueError,"refcount is not 1 after removal from dict");
return NULL;
}
/* undo the temporary resurrection */
#ifdef Py_TRACE_REFS
_Py_ForgetReference(v);
#else
v->ob_refcnt=0;
#endif
Py_INCREF(Py_None);
return Py_None;
} }
static struct PyMethodDef cc_methods[] = { static struct PyMethodDef cc_methods[] = {
{"_oid_unreferenced", (PyCFunction)cc_oid_unreferenced, METH_VARARGS,
NULL
},
{"lru_items", (PyCFunction)cc_lru_items, METH_VARARGS,
"List (oid, object) pairs from the lru list, as 2-tuples.\n"
},
{"klass_items", (PyCFunction)cc_klass_items, METH_VARARGS,
"List (oid, object) pairs of cached persistent classes.\n"
},
{"full_sweep", (PyCFunction)cc_full_sweep, METH_VARARGS, {"full_sweep", (PyCFunction)cc_full_sweep, METH_VARARGS,
"full_sweep([age]) -- Perform a full sweep of the cache\n\n" "full_sweep([age]) -- Perform a full sweep of the cache\n\n"
"Make a single pass through the cache, removing any objects that are no\n" "Make a single pass through the cache, removing any objects that are no\n"
"longer referenced, and deactivating objects that have not been\n" "longer referenced, and deactivating enough objects to bring\n"
"accessed in the number of seconds given by 'age'. " "the cache under its size limit\n"
"'age defaults to the cache age.\n" "The optional 'age' parameter is ignored.\n"
}, },
{"minimize", (PyCFunction)cc_reallyfull_sweep, METH_VARARGS, {"minimize", (PyCFunction)cc_reallyfull_sweep, METH_VARARGS,
"minimize([age]) -- Remove as many objects as possible\n\n" "minimize([age]) -- Remove as many objects as possible\n\n"
"Make multiple passes through the cache, removing any objects that are no\n" "Make multiple passes through the cache, removing any objects that are no\n"
"longer referenced, and deactivating objects that have not been\n" "longer referenced, and deactivating enough objects to bring the"
"accessed in the number of seconds given by 'age'. 'age defaults to 0.\n" " cache under its size limit\n"
"The option 'age' parameter is ignored.\n"
}, },
{"incrgc", (PyCFunction)cc_incrgc, METH_VARARGS, {"incrgc", (PyCFunction)cc_incrgc, METH_VARARGS,
"incrgc() -- Perform incremental garbage collection"}, "incrgc([n]) -- Perform incremental garbage collection\n\n"
"Some other implementations support an optional parameter 'n' which\n"
"indicates a repetition count; this value is ignored.\n"},
{"invalidate", (PyCFunction)cc_invalidate, METH_VARARGS, {"invalidate", (PyCFunction)cc_invalidate, METH_VARARGS,
"invalidate(oids) -- invalidate one, many, or all ids"}, "invalidate(oids) -- invalidate one, many, or all ids"},
{"get", (PyCFunction)cc_get, METH_VARARGS, {"get", (PyCFunction)cc_get, METH_VARARGS,
...@@ -454,115 +537,102 @@ static struct PyMethodDef cc_methods[] = { ...@@ -454,115 +537,102 @@ static struct PyMethodDef cc_methods[] = {
static ccobject * static ccobject *
newccobject(PyObject *jar, int cache_size, int cache_age) newccobject(PyObject *jar, int cache_size, int cache_age)
{ {
ccobject *self; ccobject *self;
self = PyObject_NEW(ccobject, &Cctype); UNLESS(self = PyObject_NEW(ccobject, &Cctype)) return NULL;
if (!self) self->setklassstate=self->jar=NULL;
if((self->data=PyDict_New()))
{
self->jar=jar;
Py_INCREF(jar);
UNLESS (self->setklassstate=PyObject_GetAttrString(jar, "setklassstate"))
return NULL; return NULL;
self->setklassstate = self->jar = NULL; self->cache_size=cache_size;
self->data = PyDict_New(); self->non_ghost_count=0;
if (self->data) { self->klass_count=0;
self->jar=jar; self->cache_drain_resistance=0;
Py_INCREF(jar); self->ring_lock=0;
self->setklassstate = PyObject_GetAttrString(jar, "setklassstate"); self->ring_home.next = &self->ring_home;
if (!self->setklassstate) { self->ring_home.prev = &self->ring_home;
Py_DECREF(jar); return self;
Py_DECREF(self->data);
goto error;
}
self->position = 0;
self->cache_size = cache_size;
self->cache_age = cache_age < 1 ? 1 : cache_age;
self->sum_deal = 0;
self->sum_deac = 0;
self->sum_age = 0;
self->mean_deal = 0;
self->mean_deac = 0;
self->mean_age = 0;
self->df = 1;
self->dfa = 1;
self->n = 0;
self->na = 0;
self->last_check = time(NULL);
return self;
} }
error: Py_DECREF(self);
Py_DECREF(self); return NULL;
return NULL;
} }
static void static void
cc_dealloc(ccobject *self) cc_dealloc(ccobject *self)
{ {
Py_XDECREF(self->data); Py_XDECREF(self->data);
Py_XDECREF(self->jar); Py_XDECREF(self->jar);
Py_XDECREF(self->setklassstate); Py_XDECREF(self->setklassstate);
PyObject_DEL(self); PyMem_DEL(self);
} }
static PyObject * static PyObject *
cc_getattr(ccobject *self, char *name) cc_getattr(ccobject *self, char *name)
{ {
PyObject *r; PyObject *r;
if (*name == 'c') { if(check_ring(self,"getattr")) return NULL;
if(strcmp(name, "cache_age") == 0)
return PyInt_FromLong(self->cache_age); if(*name=='c')
if(strcmp(name, "cache_size") == 0) {
return PyInt_FromLong(self->cache_size); if(strcmp(name,"cache_age")==0)
if(strcmp(name, "cache_mean_age") == 0) return PyInt_FromLong(0); /* this cache does not use this value */
return PyFloat_FromDouble(self->mean_age); if(strcmp(name,"cache_size")==0)
if(strcmp(name, "cache_mean_deal") == 0) return PyInt_FromLong(self->cache_size);
return PyFloat_FromDouble(self->mean_deal); if(strcmp(name,"cache_drain_resistance")==0)
if(strcmp(name, "cache_mean_deac") == 0) return PyInt_FromLong(self->cache_drain_resistance);
return PyFloat_FromDouble(self->mean_deac); if(strcmp(name,"cache_non_ghost_count")==0)
if(strcmp(name, "cache_df") == 0) return PyInt_FromLong(self->non_ghost_count);
return PyFloat_FromDouble(self->df); if(strcmp(name,"cache_klass_count")==0)
if(strcmp(name, "cache_dfa") == 0) return PyInt_FromLong(self->klass_count);
return PyFloat_FromDouble(self->dfa); if(strcmp(name,"cache_data")==0)
if(strcmp(name, "cache_last_gc_time") == 0) {
return PyFloat_FromDouble(self->last_check); /* now a copy of our data; the ring is too fragile */
if(strcmp(name, "cache_data") == 0) { return PyDict_Copy(self->data);
Py_INCREF(self->data);
return self->data;
} }
} }
if ((strcmp(name, "has_key") == 0) if((*name=='h' && strcmp(name, "has_key")==0) ||
|| (strcmp(name, "items") == 0) (*name=='i' && strcmp(name, "items")==0) ||
|| (strcmp(name, "keys") == 0)) (*name=='k' && strcmp(name, "keys")==0)
return PyObject_GetAttrString(self->data, name); )
return PyObject_GetAttrString(self->data, name);
r = Py_FindMethod(cc_methods, (PyObject *)self, name);
if (!r) { if((r=Py_FindMethod(cc_methods, (PyObject *)self, name)))
PyErr_Clear();
return PyObject_GetAttrString(self->data, name);
}
return r; return r;
PyErr_Clear();
return PyObject_GetAttrString(self->data, name);
} }
static int static int
cc_setattr(ccobject *self, char *name, PyObject *value) cc_setattr(ccobject *self, char *name, PyObject *value)
{ {
if (value) { if(value)
{
int v; int v;
if (strcmp(name, "cache_age") == 0) { if(strcmp(name,"cache_age")==0)
v = PyInt_AsLong(value); {
if (v == -1 && PyErr_Occurred()) /* this cache doesnt use the age */
return -1;
if (v > 0)
self->cache_age = v;
return 0; return 0;
} }
if (strcmp(name, "cache_size") == 0) { if(strcmp(name,"cache_size")==0)
v = PyInt_AsLong(value); {
if (v == -1 && PyErr_Occurred()) UNLESS(PyArg_Parse(value,"i",&v)) return -1;
return -1; self->cache_size=v;
self->cache_size = v;
return 0; return 0;
} }
}
if(strcmp(name,"cache_drain_resistance")==0)
{
UNLESS(PyArg_Parse(value,"i",&v)) return -1;
self->cache_drain_resistance=v;
return 0;
}
}
PyErr_SetString(PyExc_AttributeError, name); PyErr_SetString(PyExc_AttributeError, name);
return -1; return -1;
} }
...@@ -570,7 +640,7 @@ cc_setattr(ccobject *self, char *name, PyObject *value) ...@@ -570,7 +640,7 @@ cc_setattr(ccobject *self, char *name, PyObject *value)
static int static int
cc_length(ccobject *self) cc_length(ccobject *self)
{ {
return PyDict_Size(self->data); return PyObject_Length(self->data);
} }
static PyObject * static PyObject *
...@@ -578,39 +648,245 @@ cc_subscript(ccobject *self, PyObject *key) ...@@ -578,39 +648,245 @@ cc_subscript(ccobject *self, PyObject *key)
{ {
PyObject *r; PyObject *r;
r = PyDict_GetItem(self->data, key); if(check_ring(self,"__getitem__")) return NULL;
if (!r) {
PyErr_SetObject(PyExc_KeyError, key); UNLESS (r=(PyObject *)object_from_oid(self, key))
return NULL; {
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
} }
Py_INCREF(r);
return r; return r;
} }
static int static int
cc_ass_sub(ccobject *self, PyObject *key, PyObject *v) cc_ass_sub(ccobject *self, PyObject *key, PyObject *v)
{ {
if (v) { int result;
if (PyExtensionClass_Check(v) if(v)
|| {
(PyExtensionInstance_Check(v) if( ( PyExtensionInstance_Check(v) &&
&& (((PyExtensionClass*)(v->ob_type))->class_flags & PERSISTENT_TYPE_FLAG) &&
(((PyExtensionClass*)(v->ob_type))->class_flags (v->ob_type->tp_basicsize >= sizeof(cPersistentObject))
& PERSISTENT_TYPE_FLAG) )
&& ||
(v->ob_type->tp_basicsize >= sizeof(cPersistentObject)) PyExtensionClass_Check(v)
) )
) {
return PyDict_SetItem(self->data, key, v); PyObject *oid = PyObject_GetAttr(v,py__p_oid);
PyObject *object_again;
PyErr_SetString(PyExc_ValueError, if(!oid)
"Cache values must be persistent objects or classes."); {
return -1; return -1;
}
if(PyObject_Cmp(key,oid,&result))
{
Py_DECREF(oid);
return -1;
}
Py_DECREF(oid);
if(result)
{
PyErr_SetString(PyExc_ValueError,"key must be the same as the object's oid attribute");
return -1;
}
object_again = object_from_oid(self, key);
if(object_again)
{
if(object_again!=v)
{
Py_DECREF(object_again);
PyErr_SetString(PyExc_ValueError,"Can not re-register object under a different oid");
return -1;
}
else
{
/* re-register under the same oid - no work needed */
Py_DECREF(object_again);
return 0;
}
}
if(PyExtensionClass_Check(v))
{
if(PyDict_SetItem(self->data, key, v)) return -1;
self->klass_count++;
return 0;
}
else
{
if(((cPersistentObject*)v)->cache)
{
if(((cPersistentObject*)v)->cache==(PyObject *)self)
{
/* This object is already one of ours, which is ok.
It would be very strange if someone was trying to register the
same object under a different key */
}
else
{
/* This object is already in a different cache. */
PyErr_SetString(PyExc_ValueError, "Cache values may only be in one cache.");
return -1;
}
}
if(check_ring(self,"pre-setitem")) return -1;
if(PyDict_SetItem(self->data, key, v)) return -1;
Py_INCREF(self);
((cPersistentObject*)v)->cache = (PyObject *)self;
if(((cPersistentObject*)v)->state>=0)
{
/* insert this non-ghost object into the ring just behind the home position */
self->non_ghost_count++;
((cPersistentObject*)v)->ring.next = &self->ring_home;
((cPersistentObject*)v)->ring.prev = self->ring_home.prev;
self->ring_home.prev->next = &((cPersistentObject*)v)->ring;
self->ring_home.prev = &((cPersistentObject*)v)->ring;
}
else
{
/* steal a reference from the dictionary; ghosts have a weak reference */
Py_DECREF(v);
}
if(check_ring(self,"post-setitem")) return -1;
return 0;
}
}
else
{
PyErr_SetString(PyExc_ValueError, "Cache values must be persistent objects.");
return -1;
}
}
else
{
/* unlink this item from the ring */
if(check_ring(self,"pre-delitem")) return -1;
v = (PyObject *)object_from_oid(self,key);
if(!v) return -1;
if(PyExtensionClass_Check(v))
{
self->klass_count--;
}
else
{
if(((cPersistentObject*)v)->state>=0)
{
self->non_ghost_count--;
((cPersistentObject*)v)->ring.next->prev = ((cPersistentObject*)v)->ring.prev;
((cPersistentObject*)v)->ring.prev->next = ((cPersistentObject*)v)->ring.next;
((cPersistentObject*)v)->ring.prev = NULL;
((cPersistentObject*)v)->ring.next = NULL;
}
else
{
/* This is a ghost object, so we havent kept a reference count on it.
For it have stayed alive this long someone else must be keeping a reference
to it. Therefore we need to temporarily give it back a reference count
before calling DelItem below */
Py_INCREF(v);
}
Py_DECREF(((cPersistentObject*)v)->cache);
((cPersistentObject*)v)->cache = NULL;
}
Py_DECREF(v);
if(PyDict_DelItem(self->data, key))
{
PyErr_SetString(PyExc_RuntimeError,
"unexpectedly couldnt remove key in cc_ass_sub");
return -1;
}
if(check_ring(self,"post-delitem")) return -1;
return 0;
} }
return PyDict_DelItem(self->data, key);
} }
static int _check_ring(ccobject *self,const char *context)
{
CPersistentRing *here = &(self->ring_home);
int expected = 1+self->non_ghost_count;
int total = 0;
do
{
if(++total>(expected+10)) return 3; /* ring too big, by a large margin */
if(!here->next) return 4; /* various linking problems */
if(!here->prev) return 5;
if(!here->next->prev) return 7;
if(!here->prev->next) return 8;
if(here->prev->next!=here) return 9;
if(here->next->prev!=here) return 10;
if(!self->ring_lock)
{
/* if the ring must be locked then it only contains object other than persistent instances */
if(here!=&self->ring_home)
{
cPersistentObject *object = object_from_ring(self,here,context);
if(!object) return 12;
if(object->state==cPersistent_GHOST_STATE)
return 13;
}
}
here = here->next;
}
while(here!=&self->ring_home);
if(self->ring_lock)
{
if(total<expected) return 6; /* ring too small; too big is ok when locked */
}
else
{
if(total!=expected) return 14; /* ring size wrong, or bad ghost accounting */
}
return 0;
}
static int check_ring(ccobject *self,const char *context)
{
#ifdef MUCH_RING_CHECKING
int code=_check_ring(self,context);
if(code)
{
/*printf(stderr,"BROKEN RING (code %d) in %s, size %d\n",code,context,PyDict_Size(self->data));*/
PyErr_Format(PyExc_RuntimeError,"broken ring (code %d) in %s, size %d",code,context,PyDict_Size(self->data));
return code;
}
#endif
return 0;
}
static int
present_in_ring(ccobject *self,CPersistentRing *target)
{
CPersistentRing *here = self->ring_home.next;
while(1)
{
if(here==target)
{
return 1;
}
if(here==&self->ring_home)
{
/* back to the home position, and we didnt find it */
return 0;
}
here = here->next;
}
}
static PyMappingMethods cc_as_mapping = { static PyMappingMethods cc_as_mapping = {
(inquiry)cc_length, /*mp_length*/ (inquiry)cc_length, /*mp_length*/
(binaryfunc)cc_subscript, /*mp_subscript*/ (binaryfunc)cc_subscript, /*mp_subscript*/
...@@ -636,6 +912,10 @@ static PyTypeObject Cctype = { ...@@ -636,6 +912,10 @@ static PyTypeObject Cctype = {
(hashfunc)0, /*tp_hash*/ (hashfunc)0, /*tp_hash*/
(ternaryfunc)0, /*tp_call*/ (ternaryfunc)0, /*tp_call*/
(reprfunc)0, /*tp_str*/ (reprfunc)0, /*tp_str*/
/* Space for future expansion */
0L,0L,0L,0L,
""
}; };
static PyObject * static PyObject *
...@@ -644,9 +924,9 @@ cCM_new(PyObject *self, PyObject *args) ...@@ -644,9 +924,9 @@ cCM_new(PyObject *self, PyObject *args)
int cache_size=100, cache_age=1000; int cache_size=100, cache_age=1000;
PyObject *jar; PyObject *jar;
if (!PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age)) UNLESS(PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age))
return NULL; return NULL;
return (PyObject *)newccobject(jar, cache_size, cache_age); return (PyObject*)newccobject(jar, cache_size,cache_age);
} }
static struct PyMethodDef cCM_methods[] = { static struct PyMethodDef cCM_methods[] = {
...@@ -657,12 +937,11 @@ static struct PyMethodDef cCM_methods[] = { ...@@ -657,12 +937,11 @@ static struct PyMethodDef cCM_methods[] = {
void void
initcPickleCache(void) initcPickleCache(void)
{ {
PyObject *m; PyObject *m, *d;
Cctype.ob_type = &PyType_Type; Cctype.ob_type=&PyType_Type;
if (!ExtensionClassImported) UNLESS(ExtensionClassImported) return;
return;
m = Py_InitModule4("cPickleCache", cCM_methods, cPickleCache_doc_string, m = Py_InitModule4("cPickleCache", cCM_methods, cPickleCache_doc_string,
(PyObject*)NULL, PYTHON_API_VERSION); (PyObject*)NULL, PYTHON_API_VERSION);
...@@ -670,4 +949,15 @@ initcPickleCache(void) ...@@ -670,4 +949,15 @@ initcPickleCache(void)
py_reload = PyString_InternFromString("reload"); py_reload = PyString_InternFromString("reload");
py__p_jar = PyString_InternFromString("_p_jar"); py__p_jar = PyString_InternFromString("_p_jar");
py__p_changed = PyString_InternFromString("_p_changed"); py__p_changed = PyString_InternFromString("_p_changed");
py__p_oid = PyString_InternFromString("_p_oid");
d = PyModule_GetDict(m);
PyDict_SetItemString(d,"cache_variant",PyString_FromString("stiff/c"));
#ifdef MUCH_RING_CHECKING
PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(1));
#else
PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(0));
#endif
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7