.

wcfs/gogccrash

@@ -16,7 +16,9 @@ function runtest1() {
 		#GOGC=1 $cwd/wcfs.test -test.v -test.run 'TestZBlk|TestΔBTail|TestΔBTreeAllStructs' || break
 		#GOGC=1 $cwd/wcfs.test -test.v -test.run 'TestZBlk|TestΔBTail' || break
 		#GOGC=0 $cwd/wcfs.test -test.v -test.run 'TestZBlk|TestΔBTail' || break
-		GOGC=0 $cwd/wcfs.test -test.v -test.count=100 -test.run 'TestZBlk|TestΔBTail' || break
+		#GOGC=0 $cwd/wcfs.test -test.v -test.count=100 -test.run 'TestZBlk|TestΔBTail' || break
+		#GOGC=0 $cwd/wcfs.test -test.v -test.count=100 -test.run 'TestZBlk' || break
+		GOGC=0 $cwd/wcfs.test -test.v -test.count=100 -test.run 'TestΔBTail' || break
 	done
 }
 

wcfs/testdata/.gitignore

-/*.lock
-/*.tmp
-/*.tr[0-9]

wcfs/testdata/zblk_test_gen.py

-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
-# Copyright (C) 2018-2020  Nexedi SA and Contributors.
-#                          Kirill Smelkov <kirr@nexedi.com>
-#
-# This program is free software: you can Use, Study, Modify and Redistribute
-# it under the terms of the GNU General Public License version 3, or (at your
-# option) any later version, as published by the Free Software Foundation.
-#
-# You can also Link and Combine this program with other software covered by
-# the terms of any of the Free Software licenses or any of the Open Source
-# Initiative approved licenses and Convey the resulting work. Corresponding
-# source of such a combination shall include the source code for all other
-# software used.
-#
-# This program is distributed WITHOUT ANY WARRANTY; without even the implied
-# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-#
-# See COPYING file for full licensing terms.
-# See https://www.nexedi.com/licensing for rationale and options.
-"""zblk_test_gen.py generates test data for zblk_test.go"""
-
-from ZODB.DB import DB
-from ZODB.utils import u64
-from wendelin.bigfile.file_zodb import ZBlk0, ZBlk1, ZBigFile
-from BTrees.IOBTree import IOBTree, IOBucket
-from numpy import arange
-import os, os.path, transaction
-
-import zodbtools.test.gen_testdata  # to make time predictable
-from zodbtools.test.gen_testdata import run_with_zodb3py2_compat
-
-K = 1024
-
-def main():
-    run_with_zodb3py2_compat(main2)
-
-def main2():
-    outfs = "testdata/zblk.fs"
-    rm_f(outfs)
-    rm_f(outfs + ".index")
-    db = DB(outfs)
-    conn = db.open()
-    root = conn.root()
-
-    # one ZBigFile with two data blocks
-    root['zbigf'] = zf = ZBigFile(2*1024*K)
-    root['zblk0'] = z0 = ZBlk0()
-    root['zblk1'] = z1 = ZBlk1()    # also covers ZData
-
-    # zblk0 with small data
-    z0.setblkdata(brange32(16*K))
-    # zblk1 with head + large hole + tail.
-    # head and tail are 128K because if smaller, z1.chunktab would serialize without buckets.
-    z1ht = 128*K
-    z1.setblkdata(brange32(z1ht) + bzeros(zf.blksize - 2*z1ht) + breverse(brange32(z1ht)))
-    zf.blktab[1] = z0
-    zf.blktab[3] = z1
-
-    # make sure there is at least one bucket in z1's chunktab
-    #
-    # we need at least one bucket to verify how zblk.go handles chunktab bucket loading;
-    # more than 1 bucket just wastes space, however with only 1 bucket a tree
-    # is serialized to have leaf nodes directly in the tree state.
-    # -> verify for ==2 buckets.
-    assertIOBTreeHas2Buckets(z1.chunktab)
-
-    transaction.commit()
-
-    with open("ztestdata_zblk_test.go", "w") as f:
-        def emit(v):
-            print >>f, v
-        emit("// Code generated by %s; DO NOT EDIT." % __file__)
-        emit("package main\n")
-        emit('import "lab.nexedi.com/kirr/neo/go/zodb"\n')
-
-        emit("const zf_blksize = %d" % zf.blksize)
-        emit("const zf_size    = %d" % ((zf.blktab.maxKey()+1)*zf.blksize))
-        emit("const z0_oid     = zodb.Oid(%d)" % u64(z0._p_oid))
-        emit("const z1_oid     = zodb.Oid(%d)" % u64(z1._p_oid))
-        emit("const zf_oid     = zodb.Oid(%d)" % u64(zf._p_oid))
-        emit("const z0_rev     = zodb.Tid(0x%x)"  % u64(z0._p_serial))
-        emit("const z1_rev     = zodb.Tid(0x%x)"  % u64(z1._p_serial))
-        emit("const z0_len     = %d" % len(z0.loadblkdata()))
-        emit("const z1_htlen   = %d" % z1ht)
-
-
-    conn.close()
-    db.close()
-
-
-# brange32 returns bytes with big-endian uint32 sequence filling them.
-# returned bytes has len == size.
-def brange32(size):
-    # 0, 1, 2, ... as u32
-    return arange(0, size//4, dtype='>u4').tobytes()
-
-# bzeros returns bytes of requested size with 0 filling them.
-def bzeros(size):
-    return b'\0'*size
-
-# breverse returns bytes in the reverse order.
-def breverse(b):
-    assert isinstance(b, bytes)
-    _ = bytearray(b)
-    _.reverse()
-    return bytes(_)
-
-# assertIOBTreeHas2Buckets asserts that IOBTree has 2 buckets.
-def assertIOBTreeHas2Buckets(t):
-    assert isinstance(t, IOBTree)
-
-    # https://github.com/zopefoundation/BTrees/blob/4.5.0-1-gc8bf24e/BTrees/BTreeTemplate.c#L1087-L1109
-    _ = t.__getstate__()
-    assert len(_) == 2, (len(_), _)
-    assert len(_[0]) == 3, _[0]                   # (bucket0, key1, bucket1)
-    assert isinstance(_[0][0], IOBucket), _[0][0] # bucket0
-    assert isinstance(_[0][1], int),      _[0][2] # key1
-    assert isinstance(_[0][2], IOBucket), _[0][2] # bucket1
-    assert isinstance(_[1],    IOBucket), _[1]    # .firstbucket
-
-# rm_f is like `rm -f` in shell.
-def rm_f(path):
-    if os.path.exists(path):
-        os.remove(path)
-
-
-if __name__ == '__main__':
-    main()

wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/storage.go

@@ -23,7 +23,7 @@ package zodb
 import (
 	"context"
 	"fmt"
-	"log"
+//	"log"
 	"net/url"
 	"strings"
 	"sync"
@@ -127,9 +127,9 @@ func Open(ctx context.Context, zurl string, opt *OpenOptions) (IStorage, error)
 		// XXX 512K hardcoded (= ~ 128 · 4K-entries)
 		cache = NewCache(storDriver, 128 * 4*1024)
 
-		// FIXME teach cache for watching and remove vvv
-		log.Printf("zodb: FIXME: open %s: raw cache is not ready for invalidations" +
-			   " -> NoCache forced", zurl)
+		//// FIXME teach cache for watching and remove vvv
+		//log.Printf("zodb: FIXME: open %s: raw cache is not ready for invalidations" +
+		//	   " -> NoCache forced", zurl)
 		cache = nil
 	}
 

wcfs/zblk_test.go

-// Copyright (C) 2018-2020  Nexedi SA and Contributors.
-//                          Kirill Smelkov <kirr@nexedi.com>
-//
-// This program is free software: you can Use, Study, Modify and Redistribute
-// it under the terms of the GNU General Public License version 3, or (at your
-// option) any later version, as published by the Free Software Foundation.
-//
-// You can also Link and Combine this program with other software covered by
-// the terms of any of the Free Software licenses or any of the Open Source
-// Initiative approved licenses and Convey the resulting work. Corresponding
-// source of such a combination shall include the source code for all other
-// software used.
-//
-// This program is distributed WITHOUT ANY WARRANTY; without even the implied
-// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-//
-// See COPYING file for full licensing terms.
-// See https://www.nexedi.com/licensing for rationale and options.
-
-package main
-
-//go:generate ./testdata/zblk_test_gen.py
-
-import (
-	"bytes"
-	"context"
-	"encoding/binary"
-	"testing"
-
-	"lab.nexedi.com/kirr/go123/exc"
-	"lab.nexedi.com/kirr/neo/go/transaction"
-	"lab.nexedi.com/kirr/neo/go/zodb"
-	_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
-
-	"github.com/stretchr/testify/require"
-)
-
-const K = 1024
-
-// TestZBlk verifies that ZBlk* and ZBigFile saved by Python can be read correctly by Go.
-// TODO also test with data saved by Python3.
-func TestZBlk(t *testing.T) {
-	X := exc.Raiseif
-	assert := require.New(t)
-	ctx := context.Background()
-	stor, err := zodb.Open(ctx, "testdata/zblk.fs", &zodb.OpenOptions{ReadOnly: true}); X(err)
-	db := zodb.NewDB(stor)
-	defer func() {
-		err := db.Close();	X(err)
-		err  = stor.Close();	X(err)
-	}()
-
-	txn, ctx := transaction.New(ctx)
-	defer txn.Abort()
-
-	conn, err := db.Open(ctx, &zodb.ConnOptions{});	X(err)
-
-	xz0, err := conn.Get(ctx, z0_oid);	X(err)
-	xz1, err := conn.Get(ctx, z1_oid);	X(err)
-	xzf, err := conn.Get(ctx, zf_oid);	X(err)
-
-	z0, ok := xz0.(*ZBlk0)
-	if !ok {
-		t.Fatalf("z0: want ZBlk0; got %T", xz0)
-	}
-
-	z1, ok := xz1.(*ZBlk1)
-	if !ok {
-		t.Fatalf("z1: want ZBlk1; got %T", xz1)
-	}
-
-	zf, ok := xzf.(*ZBigFile)
-	if !ok {
-		t.Fatalf("zf: want ZBigFile; got %T", xzf)
-	}
-
-	xactivate := func(obj zodb.IPersistent) {
-		err := obj.PActivate(ctx)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-
-	z0Data, z0Rev, err := z0.loadBlkData(ctx);	X(err)
-	z0DataOK := brange32(z0_len)
-	assert.Equal(z0Data, z0DataOK,   "ZBlk0 data wrong")
-	assert.Equal(z0Rev,  z0_rev,	 "ZBlk0 rev wrong")
-
-	z1Data, z1Rev, err := z1.loadBlkData(ctx);	X(err)
-	z1DataOK := make([]byte, zf_blksize)                                  // zeros
-	copy(z1DataOK[0:],                      brange32(z1_htlen))           // head
-	copy(z1DataOK[len(z1DataOK)-z1_htlen:], breverse(brange32(z1_htlen))) // tail
-	z1DataOK = bytes.TrimRight(z1DataOK, "\x00") // trailing 0 are not persisted
-	assert.Equal(z1Data, z1DataOK,   "ZBlk1 data wrong")
-	assert.Equal(z1Rev,  z1_rev,     "ZBlk1 rev wrong")
-
-
-	xactivate(zf)
-	if zf.blksize != zf_blksize {
-		t.Fatalf("zf: blksize=%d;  want %d", zf.blksize, zf_blksize)
-	}
-
-	z0_, ok, err := zf.blktab.Get(ctx, 1);	X(err)
-	if !(ok && z0_ == z0) {
-		t.Fatalf("zf: [0] -> %#v; want z0", z0_)
-	}
-
-	z1_, ok, err := zf.blktab.Get(ctx, 3);	X(err)
-	if !(ok && z1_ == z1) {
-		t.Fatalf("zf: [1] -> %#v; want z1", z1_)
-	}
-
-	size, _, err := zf.Size(ctx);	X(err)
-	assert.Equal(size, int64(zf_size),	"ZBigFile size wrong")
-
-
-	// LoadBlk
-	z0Data, _, _, _, err = zf.LoadBlk(ctx, 1);	X(err)
-	assert.Equal(len(z0Data), int(zf.blksize))
-	z0Data = bytes.TrimRight(z0Data, "\x00")
-	assert.Equal(z0Data, z0DataOK)
-
-	z1Data, _, _, _, err = zf.LoadBlk(ctx, 3);	X(err)
-	assert.Equal(len(z1Data), int(zf.blksize))
-	z1Data = bytes.TrimRight(z1Data, "\x00")
-	assert.Equal(z1Data, z1DataOK)
-}
-
-// TODO verify PyGetState vs PySetState
-
-// brange32 returns bytes with big-endian uint32 seqence filling them.
-// returned bytes has len == size.
-func brange32(size int) []byte {
-	data := make([]byte, size)
-	for i := 0; i < size / 4; i++ {
-		binary.BigEndian.PutUint32(data[i*4:], uint32(i))
-	}
-	return data
-}
-
-// breverse returns bytes in the reverse order.
-func breverse(b []byte) []byte {
-	r := make([]byte, len(b))
-	for i := range(b) {
-		r[i] = b[len(b)-i-1]
-	}
-	return r
-}

wcfs/ztestdata_zblk_test.go

-// Code generated by ./testdata/zblk_test_gen.py; DO NOT EDIT.
-package main
-
-import "lab.nexedi.com/kirr/neo/go/zodb"
-
-const zf_blksize = 2097152
-const zf_size    = 8388608
-const z0_oid     = zodb.Oid(2)
-const z1_oid     = zodb.Oid(3)
-const zf_oid     = zodb.Oid(1)
-const z0_rev     = zodb.Tid(0x285cbac3851eb99)
-const z1_rev     = zodb.Tid(0x285cbac3851eb99)
-const z0_len     = 16384
-const z1_htlen   = 131072

wcfs/δbtail_test.go

@@ -61,6 +61,9 @@ import (
 	"lab.nexedi.com/kirr/go123/xerr"
 	"lab.nexedi.com/kirr/neo/go/transaction"
 	"lab.nexedi.com/kirr/neo/go/zodb"
+
+	// XXX -> fs1 only
+	_ "lab.nexedi.com/kirr/neo/go/zodb/wks"
 )
 
 const kInf Key = 10000 // inf key (TeX hack)