treegen_test.go 6.12 KB
Newer Older
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
// Copyright (C) 2020-2021  Nexedi SA and Contributors.
//                          Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.

Kirill Smelkov's avatar
.  
Kirill Smelkov committed
20
package zhist
Kirill Smelkov's avatar
.  
Kirill Smelkov committed
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
// treegen.go provides functionality:
//
//   - to commit a particular BTree topology into ZODB, and
//   - to generate set of random tree topologies that all correspond to particular {k->v} dict.
//
// treegen.py is used as helper for both tasks.

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"os/exec"
	"regexp"
	"strings"

	"lab.nexedi.com/kirr/go123/xerr"
	"lab.nexedi.com/kirr/neo/go/zodb"
)


// TreeGenSrv represents connection to running `treegen ...` server.
type TreeGenSrv struct {
	argv      []string
	pysrv     *exec.Cmd      // spawned `treegen ...`
	pyin      io.WriteCloser // input to pysrv
	pyoutRaw  io.ReadCloser  // output from pysrv
	pyout     *bufio.Reader  // buffered ^^^
}

// TreeSrv represents connection to running `treegen trees` server.
//
// Create it with StartTreeSrv(zurl).
// - Commit(treeTopology) -> tid
type TreeSrv struct {
	*TreeGenSrv
	zurl	  string
	treeRoot  zodb.Oid       // oid of the tree treegen works on
	head      zodb.Tid       // last made commit
}

// AllStructsSrv represents connection to running `treegen allstructs` server.
//
// Create it with StartAllStructsSrv().
// - AllStructs(maxdepth, maxsplit, n, seed, kv1, kv2)
type AllStructsSrv struct {
	*TreeGenSrv
}

// StartTreeGenSrv spawns `treegen ...` server.
func StartTreeGenSrv(argv ...string) (_ *TreeGenSrv, hello string, err error) {
	defer xerr.Contextf(&err, "treesrv %v: start", argv)

	// spawn `treegen ...`
	tg := &TreeGenSrv{argv: argv}
	tg.pysrv = exec.Command("./testprog/treegen.py", argv...)
	tg.pyin, err = tg.pysrv.StdinPipe()
	if err != nil {
		return nil, "", err
	}
	tg.pyoutRaw, err = tg.pysrv.StdoutPipe()
	if err != nil {
		return nil, "", err
	}
	tg.pyout = bufio.NewReader(tg.pyoutRaw)
	tg.pysrv.Stderr = os.Stderr // no redirection

	err = tg.pysrv.Start()
	if err != nil {
		return nil, "", err
	}

	// wait for hello message and return it
	defer func() {
		if err != nil {
			tg.Close() // ignore error
		}
	}()
	defer xerr.Context(&err, "handshake")
	hello, err = tg.pyout.ReadString('\n')
	if err != nil {
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return nil, "", err
	}
	hello = strings.TrimSuffix(hello, "\n")

	return tg, hello, nil
}

// Close shutdowns treegen server.
func (tg *TreeGenSrv) Close() (err error) {
	defer xerr.Contextf(&err, "treegen %v: close", tg.argv)

	err1 := tg.pyin.Close()
	err2 := tg.pyoutRaw.Close()
	err3 := tg.pysrv.Wait()
	return xerr.Merge(err1, err2, err3)
}


// StartTreeSrv spawns `treegen trees` server.
func StartTreeSrv(zurl string) (_ *TreeSrv, err error) {
	defer xerr.Contextf(&err, "tree.srv %s: start", zurl)
	tgSrv, hello, err := StartTreeGenSrv("trees", zurl)
	if err != nil {
		return nil, err
	}

	tg := &TreeSrv{TreeGenSrv: tgSrv, zurl: zurl}
	defer func() {
		if err != nil {
			tgSrv.Close() // ignore error
		}
	}()

	// tree.srv start @<at> tree=<root>
	defer xerr.Contextf(&err, "invalid hello %q", hello)
	startRe := regexp.MustCompile(`^tree.srv start @([^ ]+) root=([^ ]+)$`)
	m := startRe.FindStringSubmatch(hello)
	if m == nil {
		return nil, fmt.Errorf("unexpected format")
	}
	tg.head, err = zodb.ParseTid(m[1]) // <at>
	if err != nil {
		return nil, fmt.Errorf("tid: %s", err)
	}
	tg.treeRoot, err = zodb.ParseOid(m[2]) // <root>
	if err != nil {
		return nil, fmt.Errorf("root: %s", err)
	}

	return tg, nil
}

// StartAllStructsSrv spawns `treegen allstructs` server.
func StartAllStructsSrv() (_ *AllStructsSrv, err error) {
	defer xerr.Context(&err, "allstructs.srv: start")

	tgSrv, hello, err := StartTreeGenSrv("allstructs")
	if err != nil {
		return nil, err
	}

	sg := &AllStructsSrv{TreeGenSrv: tgSrv}
	defer func() {
		if err != nil {
			tgSrv.Close() // ignore error
		}
	}()

	defer xerr.Contextf(&err, "invalid hello %q", hello)
	if hello != "# allstructs.srv start" {
		return nil, fmt.Errorf("unexpected format")
	}

	return sg, nil
}


// Commit creates new commit with underlying tree changed to specified tree topology.
func (tg *TreeSrv) Commit(tree string) (_ zodb.Tid, err error) {
	defer xerr.Contextf(&err, "tree.srv %s: commit %s", tg.zurl, tree)

	_, err = io.WriteString(tg.pyin, tree + "\n")
	if err != nil {
		return zodb.InvalidTid, err
	}

	reply, err := tg.pyout.ReadString('\n')
	if err != nil {
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return zodb.InvalidTid, err
	}
	reply = strings.TrimSuffix(reply, "\n")

	tid, err := zodb.ParseTid(reply)
	if err != nil {
		return zodb.InvalidTid, fmt.Errorf("invalid reply: %s", err)
	}
	tg.head = tid
	return tid, nil
}

// AllStructs returns response from `treegen allstructs`
func (tg *AllStructsSrv) AllStructs(kv map[Key]string, maxdepth, maxsplit, n int, seed int64) (_ []string, err error) {
	req := fmt.Sprintf("%d %d %d/%d %s", maxdepth, maxsplit, n, seed, kvtxt(kv))
	defer xerr.Contextf(&err, "allstructs.srv: %s ", req)

	_, err = io.WriteString(tg.pyin, req + "\n")
	if err != nil {
		return nil, err
	}

	structv := []string{}
	for {
		reply, err := tg.pyout.ReadString('\n')
		if err != nil {
			if err == io.EOF {
				err = io.ErrUnexpectedEOF
			}
			return nil, err
		}
		reply = strings.TrimSuffix(reply, "\n")

		if reply == "# ----" {
			return structv, nil // end of response
		}
		if strings.HasPrefix(reply, "#") {
			continue // comment
		}

		structv = append(structv, reply)
	}
}