X Start of fstail

t/neo/storage/fs1/cmd/fstail/.gitignore

+/fstail

t/neo/storage/fs1/cmd/fstail/fstail.go

+// Copyright (C) 2017  Nexedi SA and Contributors.
+//                     Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 2, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+
+/*
+fstail - Tool to dump the last few transactions from a FileStorage.
+
+Format is the same as in fstail/py originally written by Jeremy Hylton:
+
+	https://github.com/zopefoundation/ZODB/blob/master/src/ZODB/scripts/fstail.py
+	https://github.com/zopefoundation/ZODB/commit/551122cc
+*/
+
+package main
+
+import (
+	"crypto/sha1"
+	"flag"
+)
+
+fsDump(w io.Writer, path string, ntxn int) error {
+	// we are not using fs1.Open here, since the file could be e.g. corrupt
+	// (same logic as in fstail/py)
+	f, err := os.Open(path)
+	// XXX err
+
+	// TODO get fSize
+
+	txnh := fs1.TxnHeader{}
+
+	// start iterating at tail.
+	// this should get EOF but read txnh.LenPrev ok.
+	err = txnh.Load(f, fSize, LoadAll)
+	if err != io.EOF {
+		if err == nil {
+			// XXX or allow this?
+			// though then, if we start not from a txn boundary - there are
+			// high chances further iteration will go wrong.
+			err = fmt.Errorf("%s @%v: reading past file end was unexpectedly successful: probably the file is being modified simultaneously",
+				path, fSize)
+		}
+		// otherwise err already has the context
+		return err
+	}
+	if txnh.LenPrev <= 0 {
+		return fmt.Errorf("%s @%v: previous record could not be read", path, fSize)
+	}
+
+	// now loop loading previous txn until EOF / ntxn limit
+	for {
+		err = txnh.LoadPrev(fs1.LoadAll)
+		if err != nil {
+			if err == io.EOF {
+				err = nil
+			}
+			break
+		}
+
+		// TODO read raw data	(get_raw_data) -> sha1
+		// txnh.Tid.TimeStamp() ?  XXX -> PerTimeStamp ? (get_timestamp)
+
+		//"user=%'q description=%'q length=%d offset=%d (+%d)", txnh.User, txnh.Description, // .Len, .offset ...
+	}
+
+	if err != nil {
+		// TODO
+	}
+}
+
+func usage() {
+	fmt.Fprintf(os.Stderr,
+`fstail [options] <storage>
+Dump transactions from a FileStorage in reverse order
+
+<storage> is a path to FileStorage
+
+  options:
+
+	-h --help       this help text.
+	-n <N>	        output the last <N> transactions (default %d).
+`)
+}
+
+func main() {
+	ntxn := 10
+	flag.IntVar(&ntxn, "n", ntxn, "output the last <N> transactions")
+	flag.Parse()
+
+	argv := flag.Args()
+	if len(argv) < 1 {
+		usage()
+		os.Exit(2)	// XXX recheck it is same as from flag.Parse on -zzz
+	}
+	storPath := argv[0]
+
+	err = fsDump(os.Stdout, storPath, n)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

t/neo/storage/fs1/filestorage.go

@@ -363,7 +363,8 @@ func (txnh *TxnHeader) LoadPrev(r io.ReaderAt, flags TxnLoadFlags) error {
 	// here we know: Load already checked txnh.Pos - lenPrev to be valid position
 	err := txnh.Load(r, txnh.Pos - lenPrev, flags)
 	if err != nil {
-		return err
+		// EOF forward is unexpected here
+		return noEOF(err)
 	}
 
 	if txnh.Len != lenPrev {

t/neo/zodb/cmd/zodbdump/zodbdump.go

@@ -307,8 +307,6 @@ func main() {
 	storUrl := argv[0]
 
 
-	var err error
-
 	if len(argv) > 1 {
 		tidRange = argv[1]
 	}

t/neo/zodb/time.go

+// TODO copyright / license
+
+// tid connection with time
+
+package zodb
+
+import (
+	"time"
+)
+
+// TimeStamp is the same as time.Time only .String() is adjusted to be the same as in ZODB/py
+// XXX get rid eventually of this
+type TimeStamp struct {
+	time.Time
+}
+
+func (t TimeStamp) String() string {
+	// NOTE UTC() in case we get TimeStamp with modified from-outside location
+	return t.UTC().Format("2006-01-02 15:04:05.000000")
+}
+
+
+
+// Time converts tid to time
+func (tid Tid) Time() TimeStamp {
+	// the same as _parseRaw in TimeStamp/py
+	// https://github.com/zopefoundation/persistent/blob/aba23595/persistent/timestamp.py#L75
+	a := uint64(tid) >> 32
+	b := uint64(tid) & (1 << 32 - 1)
+	min   := a % 60
+	hour  := a / 60 % 24
+	day   := a / (60 * 24) % 31 + 1
+	month := a / (60 * 24 * 31) % 12 + 1
+	year  := a / (60 * 24 * 31 * 12) + 1900
+	sec   := b * 60 / (1 << 32)
+	nsec  := (b - (sec << 32) / 60) * 60 * 1E9 / (1 << 32)
+
+	t := time.Date(
+		int(year),
+		time.Month(month),
+		int(day),
+		int(hour),
+		int(min),
+		int(sec),
+		int(nsec),
+		time.UTC)
+
+	// round to microsecond: zodb/py does this, and without rounding it is sometimes
+	// not exactly bit-to-bit the same in text output compared to zodb/py. Example:
+	// 037969f722a53488: timeStr = "2008-10-24 05:11:08.119999"  ; want "2008-10-24 05:11:08.120000"
+	t = t.Round(time.Microsecond)
+
+	return TimeStamp{t}
+}
+
+
+// TODO TidFromTime()
+// TODO TidFromTimeStamp()
+
+// TODO TidForNow() ?

t/neo/zodb/time_test.go

+// TODO copyright / license
+// TODO what it is
+
+package zodb
+
+import (
+	"testing"
+)
+
+func TestTidTime(t *testing.T) {
+	var testv = []struct {tid Tid; timeStr string} {
+		{0x0000000000000000, "1900-01-01 00:00:00.000000"},
+		{0x0285cbac258bf266, "1979-01-03 21:00:08.800000"},
+		{0x037969f722a53488, "2008-10-24 05:11:08.120000"},
+		{0x03b84285d71c57dd, "2016-07-01 09:41:50.416574"},
+	}
+
+	for _, tt := range testv {
+		timeStr := tt.tid.Time().String()
+		if timeStr != tt.timeStr {
+			t.Errorf("%v: timeStr = %q  ; want %q", tt.tid, timeStr, tt.timeStr)
+		}
+	}
+}