Commit 3fe09d10 authored by Carlos Ramos Carreño's avatar Carlos Ramos Carreño

Fix tests in Python 3.

Fixes the tests for Python 3, except for
lib/tests/test_zodb.py::test_zstor_2zurl which require NEO support
for Python 3.
parent 3846997b
......@@ -107,7 +107,7 @@ def test_bigarray_noobject(testbig):
# NOTE str & unicode are fixed-size types - if size is not explicitly given
# it will become S0 or U0
obj_dtypev = [numpy.object, 'O', 'i4, O', [('x', 'i4'), ('y', 'i4, O')]]
obj_dtypev = [object, 'O', 'i4, O', [('x', 'i4'), ('y', 'i4, O')]]
for dtype_ in obj_dtypev:
with raises(TypeError):
BigArray((1,), dtype_, Zh)
......
......@@ -33,7 +33,6 @@ from golang import defer, func
from pytest import raises
import pytest; xfail = pytest.mark.xfail
from ZEO.ClientStorage import ClientStorage as ZEOStorage
from neo.client.Storage import Storage as NEOStorage
import os
from six.moves.urllib.parse import quote_plus
......@@ -494,6 +493,7 @@ def test_zstor_2zurl(tmpdir, neo_ssl_dict):
# neo returns new NEO client for specified cluster name and master address.
# NOTE, similarly to ZEO, the client is returned without waiting until server nodes are connected.
def neo(cluster_name, master_addr, ssl=0):
from neo.client.Storage import Storage as NEOStorage
kwargs = dict(master_nodes=master_addr, name=cluster_name)
if ssl:
kwargs.update(neo_ssl_dict)
......
......@@ -401,7 +401,7 @@ setup(
'pytest',
'scipy',
'neoppod', # lib/tests/test_zodb.py
'ZEO', # lib/tests/test_zodb.py
'ZEO[test]', # lib/tests/test_zodb.py
],
},
......
......@@ -67,6 +67,7 @@ The following environment variables can be used to control wcfs.py client:
from __future__ import print_function, absolute_import
import os, sys, hashlib, subprocess, stat
import six
import logging; log = logging.getLogger('wcfs')
from os.path import dirname
from stat import S_ISDIR
......@@ -504,6 +505,9 @@ def _mntpt_4zurl(zurl):
# masters, we still have them associated with the same wcfs mountpoint.
zurl = zurl_normalize_main(zurl)
if isinstance(zurl, six.text_type):
zurl = zurl.encode("utf-8")
m = hashlib.sha1()
m.update(zurl)
......
......@@ -19,6 +19,7 @@
# See https://www.nexedi.com/licensing for rationale and options.
# cython: language_level=2
# cython: c_string_encoding=utf8
# distutils: language=c++
# Package _wcfs provides Python-wrappers for C++ wcfs client package.
......
......@@ -19,6 +19,7 @@
# See https://www.nexedi.com/licensing for rationale and options.
# cython: language_level=2
# cython: c_string_type=str, c_string_encoding=utf8
# cython: auto_pickle=False
# distutils: language=c++
......
......@@ -3,7 +3,9 @@
package pycompat
import (
"fmt"
"math/big"
pickle "github.com/kisielk/og-rek"
)
// Int64 tries to convert unpickled Python value to int64.
......@@ -23,3 +25,20 @@ func Int64(xv interface{}) (v int64, ok bool) {
return 0, false
}
// Xstrbytes verifies and extacts str|bytes from unpickled value.
func Xstrbytes(x interface{}) (string, error) {
var s string
switch x := x.(type) {
default:
return "", fmt.Errorf("expect str|bytes; got %T", x)
case string:
s = x
case pickle.Bytes:
s = string(x)
}
return s, nil
}
\ No newline at end of file
......@@ -18,6 +18,7 @@
# See https://www.nexedi.com/licensing for rationale and options.
# cython: language_level=2
# cython: c_string_type=str, c_string_encoding=utf8
# distutils: language=c++
"""Module wcfs_test.pyx complements wcfs_test.py with things that cannot be
......
......@@ -720,7 +720,7 @@ def __walkBFS(tree): # i[] of [](of _NodeInRange on each level)
assert isinstance(rn.node, (Tree, Bucket))
if isinstance(rn.node, Tree):
v = (rn.range.klo,) + rn.node.keyv + (rn.range.khi,)
rv = zip(v[:-1], v[1:]) # (klo,k1), (k1,k2), ..., (kN,khi)
rv = list(zip(v[:-1], v[1:])) # (klo,k1), (k1,k2), ..., (kN,khi)
assert len(rv) == len(rn.node.children)
for i in range(len(rv)):
......@@ -771,7 +771,7 @@ def __zwalkBFS(ztree): # i[] of [](of _NodeInRange on each level)
panic("bad tree kind %r" % kind)
v = [rn.range.klo] + keyv + [rn.range.khi]
rv = zip(v[:-1], v[1:]) # (klo,k1), (k1,k2), ..., (kN,khi)
rv = list(zip(v[:-1], v[1:])) # (klo,k1), (k1,k2), ..., (kN,khi)
assert len(rv) == len(children)
for i in range(len(rv)):
......
......@@ -31,6 +31,7 @@ import (
"lab.nexedi.com/kirr/neo/go/zodb"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xbtree/blib"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/pycompat"
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xzodb"
)
......@@ -302,8 +303,8 @@ func xGetBlkTab(db *zodb.DB, at zodb.Tid) map[zodb.Oid]ZBlkInfo {
defer zblkdir.PDeactivate()
for xname, xzblk := range zblkdir.Data {
name, ok := xname.(string)
if !ok {
name, err := pycompat.Xstrbytes(xname)
if err != nil {
exc.Raisef("root['treegen/values']: key [%q]: expected str, got %T", xname, xname)
}
......
......@@ -191,13 +191,14 @@ class ZCtx(object):
valdict = zctx.root.get('treegen/values', None)
if valdict is None:
valdict = zctx.root['treegen/values'] = PersistentMapping()
valv = b'abcdefghij'
valv = 'abcdefghij'
for v in valv:
v_data = v.encode('ascii')
zblk = valdict.get(v, None)
if zblk is not None and zblk.loadblkdata() == v:
if zblk is not None and zblk.loadblkdata() == v_data:
continue
zblk = ZBlk()
zblk.setblkdata(v)
zblk.setblkdata(v_data)
valdict[v] = zblk
zctx.valdict = valdict
commit('treegen/values: init %r' % valv, skipIfEmpty=True)
......@@ -259,7 +260,7 @@ def TreesSrv(zstor, r):
t, D = treetxt.split()
assert D.startswith('D')
kv = kvDecode(t[1:], zctx.vdecode)
zv = _kvDecode(D[1:], kdecode=lambda ktxt: ktxt, vdecode=lambda vtxt: vtxt)
zv = _kvDecode(D[1:], kdecode=lambda ktxt: ktxt, vdecode=lambda vtxt: vtxt.encode('ascii'))
patch(ztree, diff(ztree, kv), kv)
# ~ patch(valdict, diff(valdict,zv)) but sets zblk.value on change
......
......@@ -97,8 +97,8 @@ func (zb *zBlk0State) PyGetState() interface{} {
// PySetState implements zodb.PyStateful.
func (zb *zBlk0State) PySetState(pystate interface{}) error {
blkdata, ok := pystate.(string)
if !ok {
blkdata, err := pycompat.Xstrbytes(pystate)
if err != nil {
return fmt.Errorf("expect str; got %s", xzodb.TypeOf(pystate))
}
......@@ -143,8 +143,8 @@ func (zd *zDataState) PyGetState() interface{} {
// PySetState implements zodb.PyStateful.
func (zd *zDataState) PySetState(pystate interface{}) error {
data, ok := pystate.(string)
if !ok {
data, err := pycompat.Xstrbytes(pystate)
if err != nil {
return fmt.Errorf("expect str; got %s", xzodb.TypeOf(pystate))
}
......
......@@ -41,7 +41,8 @@ from persistent.timestamp import TimeStamp
from ZODB.utils import z64, u64, p64
import sys, os, os.path, subprocess
from thread import get_ident as gettid
import six
from six.moves._thread import get_ident as gettid
from time import gmtime
from errno import EINVAL, ENOTCONN
from resource import setrlimit, getrlimit, RLIMIT_MEMLOCK
......@@ -481,7 +482,7 @@ class tDB(tWCFS):
def change(t, zf, changeDelta):
assert isinstance(zf, ZBigFile)
zfDelta = t._changed.setdefault(zf, {})
for blk, data in changeDelta.iteritems():
for blk, data in six.iteritems(changeDelta):
data = b(data)
assert len(data) <= zf.blksize
zfDelta[blk] = data
......@@ -520,7 +521,7 @@ class tDB(tWCFS):
dfile = DFile()
zconns.add(zf._p_jar)
zfh = zf.fileh_open(_use_wcfs=False)
for blk, data in zfDelta.iteritems():
for blk, data in six.iteritems(zfDelta):
dfile.ddata[blk] = data
data += b'\0'*(zf.blksize - len(data)) # trailing \0
vma = zfh.mmap(blk, 1)
......@@ -793,7 +794,7 @@ class tFile:
try:
b = read_exfault_nogil(blkview[0:1])
except SegmentationFault:
b = 'FAULT'
b = b'FAULT'
t._blkaccess(blk)
have_read.send(b)
go(_)
......@@ -805,7 +806,7 @@ class tFile:
raise ctx.err()
b = _rx
ev.append('read ' + b)
ev.append(b'read ' + b)
ev = doCheckingPin(_, pinokByWLink, pinfunc)
# XXX hack - wlinks are notified and emit events simultaneously - we
......@@ -813,7 +814,7 @@ class tFile:
# are inside (i.e. read is stuck until pins are acknowledged).
# Better do explicit check in tracetest style.
assert ev[0] == 'read pre', ev
assert ev[-1] == 'read ' + dataok[0], ev
assert ev[-1] == b'read ' + dataok[0:1], ev
ev = ev[1:-1]
if not shouldPin:
assert ev == []
......@@ -843,7 +844,7 @@ class tFile:
assert st.st_blksize == t.blksize
assert st.st_size == len(dataokv)*t.blksize
if mtime is not None:
assert st.st_mtime == tidtime(mtime)
assert st.st_mtime == pytest.approx(tidtime(mtime))
cachev = t.cached()
for blk, dataok in enumerate(dataokv):
......@@ -1100,7 +1101,7 @@ def _expectPin(twlink, ctx, zf, expect): # -> []SrvReq
expected = set() # of expected pin messages
for blk, at in expect.items():
hat = h(at) if at is not None else 'head'
msg = b"pin %s #%d @%s" % (h(zf._p_oid), blk, hat)
msg = "pin %s #%d @%s" % (h(zf._p_oid), blk, hat)
assert msg not in expected
expected.add(msg)
......@@ -1372,7 +1373,7 @@ def test_wcfs_watch_robust():
def _(ctx):
req = wl.recvReq(ctx)
assert req is not None
assert req.msg == b"pin %s #%d @%s" % (h(zf._p_oid), 2, h(at1))
assert req.msg == "pin %s #%d @%s" % (h(zf._p_oid), 2, h(at1))
# don't reply to req - close instead
wl.closeWrite()
wg.go(_)
......@@ -1385,8 +1386,8 @@ def test_wcfs_watch_robust():
# invalid requests -> wcfs replies error
wl = t.openwatch()
assert wl.sendReq(timeout(), b'bla bla') == \
b'error bad watch: not a watch request: "bla bla"'
assert wl.sendReq(timeout(), 'bla bla') == \
'error bad watch: not a watch request: "bla bla"'
# invalid request not following frame structure -> fatal + wcfs must close watch link
assert wl.fatalv == []
......@@ -1397,7 +1398,7 @@ def test_wcfs_watch_robust():
)
if _ == 0:
raise RuntimeError("%s: did not rx EOF after bad frame " % wl)
assert wl.fatalv == [b'error: invalid frame: "zzz hello\\n" (invalid stream)']
assert wl.fatalv == ['error: invalid frame: "zzz hello\\n" (invalid stream)']
wl.close()
# watch with @at < δtail.tail -> rejected
......@@ -1476,7 +1477,7 @@ def test_wcfs_pintimeout_kill():
def _(ctx):
req = wl.recvReq(ctx)
assert req is not None
assert req.msg == b"pin %s #%d @%s" % (h(zf._p_oid), 2, h(at1))
assert req.msg == "pin %s #%d @%s" % (h(zf._p_oid), 2, h(at1))
# sleep > wcfs pin timeout - wcfs must kill us
_, _rx = select(
......@@ -1512,9 +1513,9 @@ def test_wcfs_watch_setup_ahead():
defer(wl.close)
wat = tidfromtime(tidtime(at1) + 1*dt) # > at1, but < at2
rx = wl.sendReq(ctx, b"watch %s @%s" % (h(zf._p_oid), h(wat)))
rx = wl.sendReq(ctx, "watch %s @%s" % (h(zf._p_oid), h(wat)))
assert ready(committing)
assert rx == b"ok"
assert rx == "ok"
wg.go(_)
# T2: sleep(10·dt); commit
......@@ -1738,7 +1739,7 @@ def test_wcfs_remmap_on_pin():
assert at == at1
mm.map_into_ro(f._blk(blk), f1.f.fileno(), blk*f.blksize)
f._assertBlk(2, 'hello', {wl: {2:at1}}, pinfunc=_) # NOTE not world
f._assertBlk(2, b'hello', {wl: {2:at1}}, pinfunc=_) # NOTE not world
# verify that pin message is not sent for the same blk@at twice.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment