Skip to content

Z
ZEO
Commit 100a2556 authored by Jim Fulton's avatar Jim Fulton
Browse files
Options

Fixed a compiler warning. 

Cleaned up code formatting.
parent 4f579850
Show whitespace changes
Inline Side-by-side
Showing with 1698 additions and 1570 deletions
src/persistent/cPersistence.c
View file @ 100a2556
......@@ -10,11 +10,11 @@
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
****************************************************************************/
static char cPersistence_doc_string[] =
"Defines Persistent mixin class for persistent objects.\n"
"\n"
"$Id$\n";
"Defines Persistent mixin class for persistent objects.\n"
"\n"
"$Id$\n";
#include "cPersistence.h"
#include "structmember.h"
......@@ -82,11 +82,13 @@ static void ghostify(cPersistentObject*);
static int
unghostify(cPersistentObject *self)
{
if (self->state < 0 && self->jar) {
if (self->state < 0 && self->jar)
{
PyObject *r;
/* Is it ever possible to not have a cache? */
if (self->cache) {
if (self->cache)
{
/* Create a node in the ring for this unghostified object. */
self->cache->non_ghost_count++;
self->cache->total_estimated_size +=
......@@ -100,13 +102,15 @@ unghostify(cPersistentObject *self)
self->state = cPersistent_CHANGED_STATE;
/* Call the object's __setstate__() */
r = PyObject_CallMethod(self->jar, "setstate", "O", (PyObject *)self);
if (r == NULL) {
if (r == NULL)
{
ghostify(self);
return -1;
}
self->state = cPersistent_UPTODATE_STATE;
Py_DECREF(r);
if (self->cache && self->ring.r_next == NULL) {
if (self->cache && self->ring.r_next == NULL)
{
#ifdef Py_DEBUG
fatal_1350(self, "unghostify",
"is not in the cache despite that we just "
......@@ -134,23 +138,6 @@ accessed(cPersistentObject *self)
ring_move_to_head(&self->cache->ring_home, &self->ring);
}
static void
unlink_from_ring(cPersistentObject *self)
{
/* If the cache has been cleared, then a non-ghost object
isn't in the ring any longer.
*/
if (self->ring.r_next == NULL)
return;
/* if we're ghostifying an object, we better have some non-ghosts */
assert(self->cache->non_ghost_count > 0);
self->cache->non_ghost_count--;
self->cache->total_estimated_size -=
_estimated_size_in_bytes(self->estimated_size);
ring_del(&self->ring);
}
static void
ghostify(cPersistentObject *self)
{
......@@ -161,12 +148,14 @@ ghostify(cPersistentObject *self)
return;
/* Is it ever possible to not have a cache? */
if (self->cache == NULL) {
if (self->cache == NULL)
{
self->state = cPersistent_GHOST_STATE;
return;
}
if (self->ring.r_next == NULL) {
if (self->ring.r_next == NULL)
{
/* There's no way to raise an error in this routine. */
#ifdef Py_DEBUG
fatal_1350(self, "ghostify", "claims to be in a cache but isn't");
......@@ -183,7 +172,8 @@ ghostify(cPersistentObject *self)
ring_del(&self->ring);
self->state = cPersistent_GHOST_STATE;
dictptr = _PyObject_GetDictPtr((PyObject *)self);
if (dictptr && *dictptr) {
if (dictptr && *dictptr)
{
Py_DECREF(*dictptr);
*dictptr = NULL;
}
......@@ -213,7 +203,8 @@ changed(cPersistentObject *self)
if (meth == NULL)
return -1;
arg = PyTuple_New(1);
if (arg == NULL) {
if (arg == NULL)
{
Py_DECREF(meth);
return -1;
}
......@@ -235,9 +226,11 @@ changed(cPersistentObject *self)
static PyObject *
Per__p_deactivate(cPersistentObject *self)
{
if (self->state == cPersistent_UPTODATE_STATE && self->jar) {
if (self->state == cPersistent_UPTODATE_STATE && self->jar)
{
PyObject **dictptr = _PyObject_GetDictPtr((PyObject *)self);
if (dictptr && *dictptr) {
if (dictptr && *dictptr)
{
Py_DECREF(*dictptr);
*dictptr = NULL;
}
......@@ -268,7 +261,8 @@ Per__p_invalidate(cPersistentObject *self)
{
signed char old_state = self->state;
if (old_state != cPersistent_GHOST_STATE) {
if (old_state != cPersistent_GHOST_STATE)
{
if (Per_set_changed(self, NULL) < 0)
return NULL;
ghostify(self);
......@@ -284,14 +278,16 @@ pickle_slotnames(PyTypeObject *cls)
PyObject *slotnames;
slotnames = PyDict_GetItem(cls->tp_dict, py___slotnames__);
if (slotnames) {
if (slotnames)
{
Py_INCREF(slotnames);
return slotnames;
}
slotnames = PyObject_CallFunctionObjArgs(copy_reg_slotnames,
(PyObject*)cls, NULL);
if (slotnames && !(slotnames == Py_None || PyList_Check(slotnames))) {
if (slotnames && !(slotnames == Py_None || PyList_Check(slotnames)))
{
PyErr_SetString(PyExc_TypeError,
"copy_reg._slotnames didn't return a list or None");
Py_DECREF(slotnames);
......@@ -315,8 +311,10 @@ pickle_copy_dict(PyObject *state)
if (!state)
return copy;
while (PyDict_Next(state, &pos, &key, &value)) {
if (key && PyString_Check(key)) {
while (PyDict_Next(state, &pos, &key, &value))
{
if (key && PyString_Check(key))
{
ckey = PyString_AS_STRING(key);
if (*ckey == '_' &&
(ckey[1] == 'v' || ckey[1] == 'p') &&
......@@ -337,20 +335,20 @@ pickle_copy_dict(PyObject *state)
static char pickle___getstate__doc[] =
"Get the object serialization state\n"
"\n"
"If the object has no assigned slots and has no instance dictionary, then \n"
"None is returned.\n"
"\n"
"If the object has no assigned slots and has an instance dictionary, then \n"
"the a copy of the instance dictionary is returned. The copy has any items \n"
"with names starting with '_v_' or '_p_' ommitted.\n"
"\n"
"If the object has assigned slots, then a two-element tuple is returned. \n"
"The first element is either None or a copy of the instance dictionary, \n"
"as described above. The second element is a dictionary with items \n"
"for each of the assigned slots.\n"
;
"Get the object serialization state\n"
"\n"
"If the object has no assigned slots and has no instance dictionary, then \n"
"None is returned.\n"
"\n"
"If the object has no assigned slots and has an instance dictionary, then \n"
"the a copy of the instance dictionary is returned. The copy has any items \n"
"with names starting with '_v_' or '_p_' ommitted.\n"
"\n"
"If the object has assigned slots, then a two-element tuple is returned. \n"
"The first element is either None or a copy of the instance dictionary, \n"
"as described above. The second element is a dictionary with items \n"
"for each of the assigned slots.\n"
;
static PyObject *
pickle___getstate__(PyObject *self)
......@@ -366,24 +364,28 @@ pickle___getstate__(PyObject *self)
dictp = _PyObject_GetDictPtr(self);
if (dictp)
state = pickle_copy_dict(*dictp);
else {
else
{
state = Py_None;
Py_INCREF(state);
}
if (slotnames != Py_None) {
if (slotnames != Py_None)
{
int i;
slots = PyDict_New();
if (!slots)
goto end;
for (i = 0; i < PyList_GET_SIZE(slotnames); i++) {
for (i = 0; i < PyList_GET_SIZE(slotnames); i++)
{
PyObject *name, *value;
char *cname;
name = PyList_GET_ITEM(slotnames, i);
if (PyString_Check(name)) {
if (PyString_Check(name))
{
cname = PyString_AS_STRING(name);
if (*cname == '_' &&
(cname[1] == 'v' || cname[1] == 'p') &&
......@@ -396,7 +398,8 @@ pickle___getstate__(PyObject *self)
value = PyObject_GetAttr(self, name);
if (value == NULL)
PyErr_Clear();
else {
else
{
int err = PyDict_SetItem(slots, name, value);
Py_DECREF(value);
if (err < 0)
......@@ -422,12 +425,14 @@ pickle_setattrs_from_dict(PyObject *self, PyObject *dict)
PyObject *key, *value;
Py_ssize_t pos = 0;
if (!PyDict_Check(dict)) {
if (!PyDict_Check(dict))
{
PyErr_SetString(PyExc_TypeError, "Expected dictionary");
return -1;
}
while (PyDict_Next(dict, &pos, &key, &value)) {
while (PyDict_Next(dict, &pos, &key, &value))
{
if (PyObject_SetAttr(self, key, value) < 0)
return -1;
}
......@@ -435,47 +440,52 @@ pickle_setattrs_from_dict(PyObject *self, PyObject *dict)
}
static char pickle___setstate__doc[] =
"Set the object serialization state\n\n"
"The state should be in one of 3 forms:\n\n"
"- None\n\n"
" Ignored\n\n"
"- A dictionary\n\n"
" In this case, the object's instance dictionary will be cleared and \n"
" updated with the new state.\n\n"
"- A two-tuple with a string as the first element. \n\n"
" In this case, the method named by the string in the first element will be\n"
" called with the second element.\n\n"
" This form supports migration of data formats.\n\n"
"- A two-tuple with None or a Dictionary as the first element and\n"
" with a dictionary as the second element.\n\n"
" If the first element is not None, then the object's instance dictionary \n"
" will be cleared and updated with the value.\n\n"
" The items in the second element will be assigned as attributes.\n"
;
"Set the object serialization state\n\n"
"The state should be in one of 3 forms:\n\n"
"- None\n\n"
" Ignored\n\n"
"- A dictionary\n\n"
" In this case, the object's instance dictionary will be cleared and \n"
" updated with the new state.\n\n"
"- A two-tuple with a string as the first element. \n\n"
" In this case, the method named by the string in the first element will\n"
" be called with the second element.\n\n"
" This form supports migration of data formats.\n\n"
"- A two-tuple with None or a Dictionary as the first element and\n"
" with a dictionary as the second element.\n\n"
" If the first element is not None, then the object's instance dictionary \n"
" will be cleared and updated with the value.\n\n"
" The items in the second element will be assigned as attributes.\n"
;
static PyObject *
pickle___setstate__(PyObject *self, PyObject *state)
{
PyObject *slots=NULL;
if (PyTuple_Check(state)) {
if (PyTuple_Check(state))
{
if (!PyArg_ParseTuple(state, "OO:__setstate__", &state, &slots))
return NULL;
}
if (state != Py_None) {
if (state != Py_None)
{
PyObject **dict;
dict = _PyObject_GetDictPtr(self);
if (dict) {
if (!*dict) {
if (dict)
{
if (!*dict)
{
*dict = PyDict_New();
if (!*dict)
return NULL;
}
}
if (*dict) {
if (*dict)
{
PyDict_Clear(*dict);
if (PyDict_Update(*dict, state) < 0)
return NULL;
......@@ -492,8 +502,8 @@ pickle___setstate__(PyObject *self, PyObject *state)
}
static char pickle___reduce__doc[] =
"Reduce an object to contituent parts for serialization\n"
;
"Reduce an object to contituent parts for serialization\n"
;
static PyObject *
pickle___reduce__(PyObject *self)
......@@ -502,7 +512,8 @@ pickle___reduce__(PyObject *self)
int l, i;
getnewargs = PyObject_GetAttr(self, py___getnewargs__);
if (getnewargs) {
if (getnewargs)
{
bargs = PyObject_CallFunctionObjArgs(getnewargs, NULL);
Py_DECREF(getnewargs);
if (!bargs)
......@@ -511,7 +522,8 @@ pickle___reduce__(PyObject *self)
if (l < 0)
goto end;
}
else {
else
{
PyErr_Clear();
l = 0;
}
......@@ -522,7 +534,8 @@ pickle___reduce__(PyObject *self)
Py_INCREF(self->ob_type);
PyTuple_SET_ITEM(args, 0, (PyObject*)(self->ob_type));
for (i = 0; i < l; i++) {
for (i = 0; i < l; i++)
{
Py_INCREF(PyTuple_GET_ITEM(bargs, i));
PyTuple_SET_ITEM(args, i+1, PyTuple_GET_ITEM(bargs, i));
}
......@@ -579,7 +592,21 @@ static void
Per_dealloc(cPersistentObject *self)
{
if (self->state >= 0)
unlink_from_ring(self);
{
/* If the cache has been cleared, then a non-ghost object
isn't in the ring any longer.
*/
if (self->ring.r_next != NULL)
{
/* if we're ghostifying an object, we better have some non-ghosts */
assert(self->cache->non_ghost_count > 0);
self->cache->non_ghost_count--;
self->cache->total_estimated_size -=
_estimated_size_in_bytes(self->estimated_size);
ring_del(&self->ring);
}
}
if (self->cache)
cPersistenceCAPI->percachedel(self->cache, self->oid);
Py_XDECREF(self->cache);
......@@ -619,15 +646,18 @@ convert_name(PyObject *name)
/* The Unicode to string conversion is done here because the
existing tp_setattro slots expect a string object as name
and we wouldn't want to break those. */
if (PyUnicode_Check(name)) {
if (PyUnicode_Check(name))
{
name = PyUnicode_AsEncodedString(name, NULL, NULL);
}
else
#endif
if (!PyString_Check(name)) {
if (!PyString_Check(name))
{
PyErr_SetString(PyExc_TypeError, "attribute name must be a string");
return NULL;
} else
}
else
Py_INCREF(name);
return name;
}
......@@ -648,16 +678,19 @@ unghost_getattr(const char *s)
{
if (*s++ != '_')
return 1;
if (*s == 'p') {
if (*s == 'p')
{
s++;
if (*s == '_')
return 0; /* _p_ */
else
return 1;
}
else if (*s == '_') {
else if (*s == '_')
{
s++;
switch (*s) {
switch (*s)
{
case 'c':
return strcmp(s, "class__");
case 'd':
......@@ -689,7 +722,8 @@ Per_getattro(cPersistentObject *self, PyObject *name)
goto Done;
s = PyString_AS_STRING(name);
if (unghost_getattr(s)) {
if (unghost_getattr(s))
{
if (unghostify(self) < 0)
goto Done;
accessed(self);
......@@ -713,7 +747,8 @@ Per__p_getattr(cPersistentObject *self, PyObject *name)
goto Done;
s = PyString_AS_STRING(name);
if (*s != '_' || unghost_getattr(s)) {
if (*s != '_' || unghost_getattr(s))
{
if (unghostify(self) < 0)
goto Done;
accessed(self);
......@@ -744,12 +779,14 @@ Per_setattro(cPersistentObject *self, PyObject *name, PyObject *v)
goto Done;
s = PyString_AS_STRING(name);
if (strncmp(s, "_p_", 3) != 0) {
if (strncmp(s, "_p_", 3) != 0)
{
if (unghostify(self) < 0)
goto Done;
accessed(self);
if (strncmp(s, "_v_", 3) != 0
&& self->state != cPersistent_CHANGED_STATE) {
&& self->state != cPersistent_CHANGED_STATE)
{
if (changed(self) < 0)
goto Done;
}
......@@ -773,14 +810,16 @@ Per_p_set_or_delattro(cPersistentObject *self, PyObject *name, PyObject *v)
goto Done;
s = PyString_AS_STRING(name);
if (strncmp(s, "_p_", 3)) {
if (strncmp(s, "_p_", 3))
{
if (unghostify(self) < 0)
goto Done;
accessed(self);
result = 0;
}
else {
else
{
if (PyObject_GenericSetAttr((PyObject *)self, name, v) < 0)
goto Done;
result = 1;
......@@ -828,7 +867,8 @@ Per__p_delattr(cPersistentObject *self, PyObject *name)
static PyObject *
Per_get_changed(cPersistentObject *self)
{
if (self->state < 0) {
if (self->state < 0)
{
Py_INCREF(Py_None);
return Py_None;
}
......@@ -841,7 +881,8 @@ Per_set_changed(cPersistentObject *self, PyObject *v)
int deactivate = 0;
int true;
if (!v) {
if (!v)
{
/* delattr is used to invalidate an object even if it has changed. */
if (self->state != cPersistent_GHOST_STATE)
self->state = cPersistent_UPTODATE_STATE;
......@@ -850,7 +891,8 @@ Per_set_changed(cPersistentObject *self, PyObject *v)
else if (v == Py_None)
deactivate = 1;
if (deactivate) {
if (deactivate)
{
PyObject *res, *meth;
meth = PyObject_GetAttr((PyObject *)self, py__p_deactivate);
if (meth == NULL)
......@@ -858,7 +900,8 @@ Per_set_changed(cPersistentObject *self, PyObject *v)
res = PyObject_CallObject(meth, NULL);
if (res)
Py_DECREF(res);
else {
else
{
/* an error occured in _p_deactivate().
It's not clear what we should do here. The code is
......@@ -884,8 +927,10 @@ Per_set_changed(cPersistentObject *self, PyObject *v)
true = PyObject_IsTrue(v);
if (true == -1)
return -1;
if (true) {
if (self->state < 0) {
if (true)
{
if (self->state < 0)
{
if (unghostify(self) < 0)
return -1;
}
......@@ -911,17 +956,20 @@ Per_get_oid(cPersistentObject *self)
static int
Per_set_oid(cPersistentObject *self, PyObject *v)
{
if (self->cache) {
if (self->cache)
{
int result;
if (v == NULL) {
if (v == NULL)
{
PyErr_SetString(PyExc_ValueError,
"can't delete _p_oid of cached object");
return -1;
}
if (PyObject_Cmp(self->oid, v, &result) < 0)
return -1;
if (result) {
if (result)
{
PyErr_SetString(PyExc_ValueError,
"can not change _p_oid of cached object");
return -1;
......@@ -944,17 +992,20 @@ Per_get_jar(cPersistentObject *self)
static int
Per_set_jar(cPersistentObject *self, PyObject *v)
{
if (self->cache) {
if (self->cache)
{
int result;
if (v == NULL) {
if (v == NULL)
{
PyErr_SetString(PyExc_ValueError,
"can't delete _p_jar of cached object");
return -1;
}
if (PyObject_Cmp(self->jar, v, &result) < 0)
return -1;
if (result) {
if (result)
{
PyErr_SetString(PyExc_ValueError,
"can not change _p_jar of cached object");
return -1;
......@@ -975,15 +1026,18 @@ Per_get_serial(cPersistentObject *self)
static int
Per_set_serial(cPersistentObject *self, PyObject *v)
{
if (v) {
if (v)
{
if (PyString_Check(v) && PyString_GET_SIZE(v) == 8)
memcpy(self->serial, PyString_AS_STRING(v), 8);
else {
else
{
PyErr_SetString(PyExc_ValueError,
"_p_serial must be an 8-character string");
return -1;
}
} else
}
else
memset(self->serial, 0, 8);
return 0;
}
......@@ -998,7 +1052,8 @@ Per_get_mtime(cPersistentObject *self)
accessed(self);
if (memcmp(self->serial, "\0\0\0\0\0\0\0\0", 8) == 0) {
if (memcmp(self->serial, "\0\0\0\0\0\0\0\0", 8) == 0)
{
Py_INCREF(Py_None);
return Py_None;
}
......@@ -1026,22 +1081,27 @@ Per_get_estimated_size(cPersistentObject *self)
static int
Per_set_estimated_size(cPersistentObject *self, PyObject *v)
{
if (v) {
if (PyInt_Check(v)) {
if (v)
{
if (PyInt_Check(v))
{
long lv = PyInt_AS_LONG(v);
if (lv < 0) {
if (lv < 0)
{
PyErr_SetString(PyExc_ValueError,
"_p_estimated_size must not be negative");
return -1;
}
self->estimated_size = _estimated_size_in_24_bits(lv);
}
else {
else
{
PyErr_SetString(PyExc_ValueError,
"_p_estimated_size must be an integer");
return -1;
}
} else
}
else
self->estimated_size = 0;
return 0;
}
......@@ -1172,12 +1232,13 @@ simple_new(PyObject *self, PyObject *type_object)
return PyType_GenericNew((PyTypeObject *)type_object, NULL, NULL);
}
static PyMethodDef cPersistence_methods[] = {
static PyMethodDef cPersistence_methods[] =
{
{"simple_new", simple_new, METH_O,
"Create an object by simply calling a class's __new__ method without "
"arguments."},
{NULL, NULL}
};
};
static cPersistenceCAPIstruct
......@@ -1241,18 +1302,21 @@ initcPersistence(void)
return;
copy_reg_slotnames = PyObject_GetAttrString(copy_reg, "_slotnames");
if (!copy_reg_slotnames) {
if (!copy_reg_slotnames)
{
Py_DECREF(copy_reg);
return;
}
__newobj__ = PyObject_GetAttrString(copy_reg, "__newobj__");
if (!__newobj__) {
if (!__newobj__)
{
Py_DECREF(copy_reg);
return;
}
if (!TimeStamp) {
if (!TimeStamp)
{
m = PyImport_ImportModule("persistent.TimeStamp");
if (!m)
return;
......
src/persistent/cPickleCache.c
View file @ 100a2556
/*****************************************************************************
/*****************************************************************************
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
......@@ -10,87 +10,87 @@
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
****************************************************************************/
/*
Objects are stored under three different regimes:
Regime 1: Persistent Classes
Persistent Classes are part of ZClasses. They are stored in the
self->data dictionary, and are never garbage collected.
The klass_items() method returns a sequence of (oid,object) tuples for
every Persistent Class, which should make it possible to implement
garbage collection in Python if necessary.
Regime 2: Ghost Objects
There is no benefit to keeping a ghost object which has no external
references, therefore a weak reference scheme is used to ensure that
ghost objects are removed from memory as soon as possible, when the
last external reference is lost.
Ghost objects are stored in the self->data dictionary. Normally a
dictionary keeps a strong reference on its values, however this
reference count is 'stolen'.
This weak reference scheme leaves a dangling reference, in the
dictionary, when the last external reference is lost. To clean up this
dangling reference the persistent object dealloc function calls
self->cache->_oid_unreferenced(self->oid). The cache looks up the oid
in the dictionary, ensures it points to an object whose reference
count is zero, then removes it from the dictionary. Before removing
the object from the dictionary it must temporarily resurrect the
object in much the same way that class instances are resurrected
before their __del__ is called.
Since ghost objects are stored under a different regime to non-ghost
objects, an extra ghostify function in cPersistenceAPI replaces
self->state=GHOST_STATE assignments that were common in other
persistent classes (such as BTrees).
Regime 3: Non-Ghost Objects
Non-ghost objects are stored in two data structures: the dictionary
mapping oids to objects and a doubly-linked list that encodes the
order in which the objects were accessed. The dictionary reference is
borrowed, as it is for ghosts. The list reference is a new reference;
the list stores recently used objects, even if they are otherwise
unreferenced, to avoid loading the object from the database again.
The doubly-link-list nodes contain next and previous pointers linking
together the cache and all non-ghost persistent objects.
The node embedded in the cache is the home position. On every
attribute access a non-ghost object will relink itself just behind the
home position in the ring. Objects accessed least recently will
eventually find themselves positioned after the home position.
Occasionally other nodes are temporarily inserted in the ring as
position markers. The cache contains a ring_lock flag which must be
set and unset before and after doing so. Only if the flag is unset can
the cache assume that all nodes are either his own home node, or nodes
from persistent objects. This assumption is useful during the garbage
collection process.
The number of non-ghost objects is counted in self->non_ghost_count.
The garbage collection process consists of traversing the ring, and
deactivating (that is, turning into a ghost) every object until
self->non_ghost_count is down to the target size, or until it
reaches the home position again.
Note that objects in the sticky or changed states are still kept in
the ring, however they can not be deactivated. The garbage collection
process must skip such objects, rather than deactivating them.
Objects are stored under three different regimes:
Regime 1: Persistent Classes
Persistent Classes are part of ZClasses. They are stored in the
self->data dictionary, and are never garbage collected.
The klass_items() method returns a sequence of (oid,object) tuples for
every Persistent Class, which should make it possible to implement
garbage collection in Python if necessary.
Regime 2: Ghost Objects
There is no benefit to keeping a ghost object which has no external
references, therefore a weak reference scheme is used to ensure that
ghost objects are removed from memory as soon as possible, when the
last external reference is lost.
Ghost objects are stored in the self->data dictionary. Normally a
dictionary keeps a strong reference on its values, however this
reference count is 'stolen'.
This weak reference scheme leaves a dangling reference, in the
dictionary, when the last external reference is lost. To clean up this
dangling reference the persistent object dealloc function calls
self->cache->_oid_unreferenced(self->oid). The cache looks up the oid
in the dictionary, ensures it points to an object whose reference
count is zero, then removes it from the dictionary. Before removing
the object from the dictionary it must temporarily resurrect the
object in much the same way that class instances are resurrected
before their __del__ is called.
Since ghost objects are stored under a different regime to non-ghost
objects, an extra ghostify function in cPersistenceAPI replaces
self->state=GHOST_STATE assignments that were common in other
persistent classes (such as BTrees).
Regime 3: Non-Ghost Objects
Non-ghost objects are stored in two data structures: the dictionary
mapping oids to objects and a doubly-linked list that encodes the
order in which the objects were accessed. The dictionary reference is
borrowed, as it is for ghosts. The list reference is a new reference;
the list stores recently used objects, even if they are otherwise
unreferenced, to avoid loading the object from the database again.
The doubly-link-list nodes contain next and previous pointers linking
together the cache and all non-ghost persistent objects.
The node embedded in the cache is the home position. On every
attribute access a non-ghost object will relink itself just behind the
home position in the ring. Objects accessed least recently will
eventually find themselves positioned after the home position.
Occasionally other nodes are temporarily inserted in the ring as
position markers. The cache contains a ring_lock flag which must be
set and unset before and after doing so. Only if the flag is unset can
the cache assume that all nodes are either his own home node, or nodes
from persistent objects. This assumption is useful during the garbage
collection process.
The number of non-ghost objects is counted in self->non_ghost_count.
The garbage collection process consists of traversing the ring, and
deactivating (that is, turning into a ghost) every object until
self->non_ghost_count is down to the target size, or until it
reaches the home position again.
Note that objects in the sticky or changed states are still kept in
the ring, however they can not be deactivated. The garbage collection
process must skip such objects, rather than deactivating them.
*/
static char cPickleCache_doc_string[] =
"Defines the PickleCache used by ZODB Connection objects.\n"
"\n"
"$Id$\n";
"Defines the PickleCache used by ZODB Connection objects.\n"
"\n"
"$Id$\n";
#define DONT_USE_CPERSISTENCECAPI
#include "cPersistence.h"
......@@ -99,6 +99,12 @@ static char cPickleCache_doc_string[] =
#include <stddef.h>
#undef Py_FindMethod
/* Python 2.4 backward compat */
#if PY_MAJOR_VERSION <= 2 && PY_MINOR_VERSION < 5
#define Py_ssize_t int
typedef Py_ssize_t (*lenfunc)(PyObject *);
#endif
/* Python string objects to speed lookups; set by module init. */
static PyObject *py__p_changed;
static PyObject *py__p_deactivate;
......@@ -116,7 +122,8 @@ typedef struct {
PyObject *data; /* oid -> object dict */
PyObject *jar; /* Connection object */
int cache_size; /* target number of items in cache */
PY_LONG_LONG cache_size_bytes; /* target total estimated size of items in cache */
PY_LONG_LONG cache_size_bytes; /* target total estimated size of
items in cache */
/* Most of the time the ring contains only:
* many nodes corresponding to persistent objects
......@@ -194,7 +201,8 @@ scan_gc_items(ccobject *self, int target, PY_LONG_LONG target_bytes)
(self->non_ghost_count > target
|| (target_bytes && self->total_estimated_size > target_bytes)
)
) {
)
{
assert(self->ring_lock);
assert(here != &self->ring_home);
......@@ -205,7 +213,8 @@ scan_gc_items(ccobject *self, int target, PY_LONG_LONG target_bytes)
is not the home */
object = OBJECT_FROM_RING(self, here);
if (object->state == cPersistent_UPTODATE_STATE) {
if (object->state == cPersistent_UPTODATE_STATE)
{
CPersistentRing placeholder;
PyObject *method;
PyObject *temp;
......@@ -227,7 +236,8 @@ scan_gc_items(ccobject *self, int target, PY_LONG_LONG target_bytes)
method = PyObject_GetAttr((PyObject *)object, py__p_deactivate);
if (method == NULL)
error_occurred = 1;
else {
else
{
temp = PyObject_CallObject(method, NULL);
Py_DECREF(method);
if (temp == NULL)
......@@ -255,13 +265,15 @@ lockgc(ccobject *self, int target_size, PY_LONG_LONG target_size_bytes)
* in between checking the ring_lock and acquiring it that calls back
* into Python.
*/
if (self->ring_lock) {
if (self->ring_lock)
{
Py_INCREF(Py_None);
return Py_None;
}
self->ring_lock = 1;
if (scan_gc_items(self, target_size, target_size_bytes) < 0) {
if (scan_gc_items(self, target_size, target_size_bytes) < 0)
{
self->ring_lock = 0;
return NULL;
}
......@@ -279,7 +291,8 @@ cc_incrgc(ccobject *self, PyObject *args)
int target_size = self->cache_size;
PY_LONG_LONG target_size_bytes = self->cache_size_bytes;
if (self->cache_drain_resistance >= 1) {
if (self->cache_drain_resistance >= 1)
{
/* This cache will gradually drain down to a small size. Check
a (small) number of objects proportional to the current size */
......@@ -360,7 +373,8 @@ _invalidate(ccobject *self, PyObject *key)
}
}
if (v->ob_refcnt <= 1 && PyType_Check(v)) {
if (v->ob_refcnt <= 1 && PyType_Check(v))
{
/* This looks wrong, but it isn't. We use strong references to types
because they don't have the ring members.
......@@ -395,19 +409,22 @@ cc_invalidate(ccobject *self, PyObject *inv)
}
PyDict_Clear(inv);
}
else {
else
{
if (PyString_Check(inv))
{
if (_invalidate(self, inv) < 0)
return NULL;
}
else {
else
{
int l, r;
l = PyObject_Length(inv);
if (l < 0)
return NULL;
for (i=l; --i >= 0; ) {
for (i=l; --i >= 0; )
{
key = PySequence_GetItem(inv, i);
if (!key)
return NULL;
......@@ -434,7 +451,8 @@ cc_get(ccobject *self, PyObject *args)
return NULL;
r = PyDict_GetItem(self->data, key);
if (!r) {
if (!r)
{
if (d)
r = d;
else
......@@ -460,14 +478,18 @@ cc_klass_items(ccobject *self)
if (l == NULL)
return NULL;
while (PyDict_Next(self->data, &p, &k, &v)) {
if(PyType_Check(v)) {
while (PyDict_Next(self->data, &p, &k, &v))
{
if(PyType_Check(v))
{
v = Py_BuildValue("OO", k, v);
if (v == NULL) {
if (v == NULL)
{
Py_DECREF(l);
return NULL;
}
if (PyList_Append(l, v) < 0) {
if (PyList_Append(l, v) < 0)
{
Py_DECREF(v);
Py_DECREF(l);
return NULL;
......@@ -524,7 +546,8 @@ cc_lru_items(ccobject *self)
PyObject *l;
CPersistentRing *here;
if (self->ring_lock) {
if (self->ring_lock)
{
/* When the ring lock is held, we have no way of know which
ring nodes belong to persistent objects, and which a
placeholders. */
......@@ -538,20 +561,24 @@ cc_lru_items(ccobject *self)
return NULL;
here = self->ring_home.r_next;
while (here != &self->ring_home) {
while (here != &self->ring_home)
{
PyObject *v;
cPersistentObject *object = OBJECT_FROM_RING(self, here);
if (object == NULL) {
if (object == NULL)
{
Py_DECREF(l);
return NULL;
}
v = Py_BuildValue("OO", object->oid, object);
if (v == NULL) {
if (v == NULL)
{
Py_DECREF(l);
return NULL;
}
if (PyList_Append(l, v) < 0) {
if (PyList_Append(l, v) < 0)
{
Py_DECREF(v);
Py_DECREF(l);
return NULL;
......@@ -645,11 +672,13 @@ cc_update_object_size_estimation(ccobject *self, PyObject *args)
return NULL;
/* Note: reference borrowed */
v = (cPersistentObject *)PyDict_GetItem(self->data, oid);
if (v) {
if (v)
{
/* we know this object -- update our "total_size_estimation"
we must only update when the object is in the ring
*/
if (v->ring.r_next) {
if (v->ring.r_next)
{
self->total_estimated_size += _estimated_size_in_bytes(
_estimated_size_in_24_bits(new_size) - v->estimated_size
);
......@@ -692,7 +721,9 @@ static struct PyMethodDef cc_methods[] = {
{"update_object_size_estimation",
(PyCFunction)cc_update_object_size_estimation,
METH_VARARGS,
"update_object_size_estimation(oid, new_size) -- update the caches size estimation for *oid* (if this is known to the cache)."},
"update_object_size_estimation(oid, new_size) -- "
"update the caches size estimation for *oid* "
"(if this is known to the cache)."},
{NULL, NULL} /* sentinel */
};
......@@ -708,7 +739,8 @@ cc_init(ccobject *self, PyObject *args, PyObject *kwds)
self->jar = NULL;
self->data = PyDict_New();
if (self->data == NULL) {
if (self->data == NULL)
{
Py_DECREF(self);
return -1;
}
......@@ -769,11 +801,13 @@ cc_clear(ccobject *self)
*/
assert(! self->ring_lock);
while (self->ring_home.r_next != &self->ring_home) {
while (self->ring_home.r_next != &self->ring_home)
{
CPersistentRing *here = self->ring_home.r_next;
cPersistentObject *o = OBJECT_FROM_RING(self, here);
if (o->cache) {
if (o->cache)
{
Py_INCREF(o); /* account for uncounted reference */
if (PyDict_DelItem(self->data, o->oid) < 0)
return -1;
......@@ -789,7 +823,8 @@ cc_clear(ccobject *self)
Py_XDECREF(self->jar);
while (PyDict_Next(self->data, &pos, &k, &v)) {
while (PyDict_Next(self->data, &pos, &k, &v))
{
Py_INCREF(v);
if (PyDict_SetItem(self->data, k, Py_None) < 0)
return -1;
......@@ -834,7 +869,8 @@ cc_traverse(ccobject *self, visitproc visit, void *arg)
if (!here)
return 0;
while (here != &self->ring_home) {
while (here != &self->ring_home)
{
cPersistentObject *o = OBJECT_FROM_RING(self, here);
VISIT(o);
here = here->r_next;
......@@ -844,7 +880,7 @@ cc_traverse(ccobject *self, visitproc visit, void *arg)
return 0;
}
static int
static Py_ssize_t
cc_length(ccobject *self)
{
return PyObject_Length(self->data);
......@@ -856,7 +892,8 @@ cc_subscript(ccobject *self, PyObject *key)
PyObject *r;
r = PyDict_GetItem(self->data, key);
if (r == NULL) {
if (r == NULL)
{
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
}
......@@ -873,10 +910,12 @@ cc_add_item(ccobject *self, PyObject *key, PyObject *v)
cPersistentObject *p;
/* Sanity check the value given to make sure it is allowed in the cache */
if (PyType_Check(v)) {
if (PyType_Check(v))
{
/* Its a persistent class, such as a ZClass. Thats ok. */
}
else if (v->ob_type->tp_basicsize < sizeof(cPersistentObject)) {
else if (v->ob_type->tp_basicsize < sizeof(cPersistentObject))
{
/* If it's not an instance of a persistent class, (ie Python
classes that derive from persistent.Persistent, BTrees,
etc), report an error.
......@@ -894,7 +933,8 @@ cc_add_item(ccobject *self, PyObject *key, PyObject *v)
oid = PyObject_GetAttr(v, py__p_oid);
if (oid == NULL)
return -1;
if (! PyString_Check(oid)) {
if (! PyString_Check(oid))
{
PyErr_Format(PyExc_TypeError,
"Cached object oid must be a string, not a %s",
oid->ob_type->tp_name);
......@@ -905,12 +945,14 @@ cc_add_item(ccobject *self, PyObject *key, PyObject *v)
* now check if they are the same string.
*/
result = PyObject_Compare(key, oid);
if (PyErr_Occurred()) {
if (PyErr_Occurred())
{
Py_DECREF(oid);
return -1;
}
Py_DECREF(oid);
if (result) {
if (result)
{
PyErr_SetString(PyExc_ValueError, "Cache key does not match oid");
return -1;
}
......@@ -919,7 +961,8 @@ cc_add_item(ccobject *self, PyObject *key, PyObject *v)
jar = PyObject_GetAttr(v, py__p_jar);
if (jar == NULL)
return -1;
if (jar==Py_None) {
if (jar==Py_None)
{
Py_DECREF(jar);
PyErr_SetString(PyExc_ValueError,
"Cached object jar missing");
......@@ -928,25 +971,33 @@ cc_add_item(ccobject *self, PyObject *key, PyObject *v)
Py_DECREF(jar);
object_again = PyDict_GetItem(self->data, key);
if (object_again) {
if (object_again != v) {
if (object_again)
{
if (object_again != v)
{
PyErr_SetString(PyExc_ValueError,
"A different object already has the same oid");
return -1;
} else {
}
else
{
/* re-register under the same oid - no work needed */
return 0;
}
}
if (PyType_Check(v)) {
if (PyType_Check(v))
{
if (PyDict_SetItem(self->data, key, v) < 0)
return -1;
self->klass_count++;
return 0;
} else {
}
else
{
PerCache *cache = ((cPersistentObject *)v)->cache;
if (cache) {
if (cache)
{
if (cache != (PerCache *)self)
/* This object is already in a different cache. */
PyErr_SetString(PyExc_ValueError,
......@@ -969,7 +1020,8 @@ cc_add_item(ccobject *self, PyObject *key, PyObject *v)
p = (cPersistentObject *)v;
Py_INCREF(self);
p->cache = (PerCache *)self;
if (p->state >= 0) {
if (p->state >= 0)
{
/* insert this non-ghost object into the ring just
behind the home position. */
self->non_ghost_count++;
......@@ -988,21 +1040,28 @@ cc_del_item(ccobject *self, PyObject *key)
/* unlink this item from the ring */
v = PyDict_GetItem(self->data, key);
if (v == NULL) {
if (v == NULL)
{
PyErr_SetObject(PyExc_KeyError, key);
return -1;
}
if (PyType_Check(v)) {
if (PyType_Check(v))
{
self->klass_count--;
} else {
}
else
{
p = (cPersistentObject *)v;
if (p->state >= 0) {
if (p->state >= 0)
{
self->non_ghost_count--;
ring_del(&p->ring);
/* The DelItem below will account for the reference
held by the list. */
} else {
}
else
{
/* This is a ghost object, so we haven't kept a reference
count on it. For it have stayed alive this long
someone else must be keeping a reference to
......@@ -1015,7 +1074,8 @@ cc_del_item(ccobject *self, PyObject *key)
p->cache = NULL;
}
if (PyDict_DelItem(self->data, key) < 0) {
if (PyDict_DelItem(self->data, key) < 0)
{
PyErr_SetString(PyExc_RuntimeError,
"unexpectedly couldn't remove key in cc_ass_sub");
return -1;
......@@ -1027,7 +1087,8 @@ cc_del_item(ccobject *self, PyObject *key)
static int
cc_ass_sub(ccobject *self, PyObject *key, PyObject *v)
{
if (!PyString_Check(key)) {
if (!PyString_Check(key))
{
PyErr_Format(PyExc_TypeError,
"cPickleCache key must be a string, not a %s",
key->ob_type->tp_name);
......@@ -1039,11 +1100,12 @@ cc_ass_sub(ccobject *self, PyObject *key, PyObject *v)
return cc_del_item(self, key);
}
static PyMappingMethods cc_as_mapping = {
(inquiry)cc_length, /*mp_length*/
static PyMappingMethods cc_as_mapping =
{
(lenfunc)cc_length, /*mp_length*/
(binaryfunc)cc_subscript, /*mp_subscript*/
(objobjargproc)cc_ass_sub, /*mp_ass_subscript*/
};
};
static PyObject *
cc_cache_data(ccobject *self, void *context)
......@@ -1051,10 +1113,11 @@ cc_cache_data(ccobject *self, void *context)
return PyDict_Copy(self->data);
}
static PyGetSetDef cc_getsets[] = {
static PyGetSetDef cc_getsets[] =
{
{"cache_data", (getter)cc_cache_data},
{NULL}
};
};
static PyMemberDef cc_members[] = {
......@@ -1126,7 +1189,8 @@ initcPickleCache(void)
Cctype.ob_type = &PyType_Type;
Cctype.tp_new = &PyType_GenericNew;
if (PyType_Ready(&Cctype) < 0) {
if (PyType_Ready(&Cctype) < 0)
{
return;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7