// Copyright (C) 2017-2021  Nexedi SA and Contributors.
//                          Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.

package neo
// infrastructure for creating NEO test clusters.

import (
	"context"
	"fmt"
	"sync"
	"testing"

	"lab.nexedi.com/kirr/go123/tracing/tracetest"
	"lab.nexedi.com/kirr/go123/xnet"
	"lab.nexedi.com/kirr/go123/xnet/lonet"
	"lab.nexedi.com/kirr/go123/xnet/pipenet"
	"lab.nexedi.com/kirr/go123/xnet/virtnet"
	"lab.nexedi.com/kirr/go123/xsync"

	"lab.nexedi.com/kirr/neo/go/internal/xcontext"
	"lab.nexedi.com/kirr/neo/go/neo/proto"

	"lab.nexedi.com/kirr/neo/go/neo/storage"
	"lab.nexedi.com/kirr/neo/go/zodb"
)

// tEnv is environment for a NEO test.
type tEnv struct {
	*tracetest.T // tracetest environment behind tEnv

	opt tClusterOptions // by default NEO clusters are created with this options
}

// tClusterOptions represents options for a test NEO cluster.
type tClusterOptions struct {
	// Network forces usage of this kind of network for node inter-networking (pipenet|lonet).
	Network string
}

// Verify verifies f for all combinations of NEO environments.
func Verify(t *testing.T, f func(*tEnv)) {
	// TODO verify M=(go|py) x S=(go|py) x ...
	// for now we only verify for all combinations of network

	// TODO verify enc=(M|N) ?

	// for all networks
	for _, network := range []string{"pipenet", "lonet"} {
		opt := tClusterOptions{
			Network: network,
		}

		t.Run("net="+network, func (t *testing.T) {
			tracetest.Verify(t, func(t *tracetest.T) {
				tenv := &tEnv{
					T:   t,
					opt: opt,
				}
				f(tenv)
			})
		})
	}
}

// ----------------------------------------

// tCluster is a test NEO cluster.
//
// Create it with tEnv.NewCluster.
// Create nodes with .NewMaster, .NewStorage and .NewClient.
//
// The cluster is tested based on synchronous event tracing -
// see package lab.nexedi.com/kirr/go123/tracing/tracetest for context.
//
// NOTE network addresses are predictable due to using pipenet/lonet for inter-node networking.
type tCluster struct {
	*tEnv // original testing env this cluster was created at

	name	string              // name of the cluster
	network *virtnet.SubNetwork // nodes interoperate via network

	traceOn  bool         // whether tracing is currently on or off
	gotracer *GoTracer    // for tracing go nodes
	//tpy	 *PyTracer    // for tracing py nodes  TODO
	erouter  *EventRouter // to which stream an event should go

	tabMu     sync.Mutex
	nodeTab	  map[string/*node*/]*tNode

	runCtx    context.Context  // runCtx is canceled on .Stop()  TODO or test failure
	runWG     *xsync.WorkGroup // started nodes are .Run() under runWG
	runCancel func()
}

// tNode represents information about a tested node.
type tNode struct {
	itest interface{}    // ITest{Master,Storage,Client}
	net   *xnet.Tracer
}

// ITestMaster represents tested master node.
type ITestMaster interface {
	Start() error
}

// ITestStorage represents tested storage node.
type ITestStorage interface {
}

// ITestClient represents tested client node.
type ITestClient interface {
	zodb.IStorageDriver
}


// NewCluster creates new NEO test cluster.
//
// At the end of the test the cluster has to be stopped via t.Stop().
//
// TODO add options describing whether node type X should be created via Go or via Py.
func (t0 *tEnv) NewCluster(name string) *tCluster {
	t := &tCluster{
		tEnv: t0,
		name: name,

		nodeTab: make(map[string]*tNode),
	}

	// create test network
	network := t0.opt.Network
	if network == "" {
		network = "pipenet" // TODO default to lonet if Py nodes will be present
	}
	var net *virtnet.SubNetwork
	var err error
	switch network {
	default:
		panic(fmt.Sprintf("unknown network %q", network))
	case "pipenet":
		net = pipenet.AsVirtNet(pipenet.New("testnet"))
	case "lonet":
		net, err = lonet.Join(context.Background(), "") // XXX ctx -> t.Ctx() ?
		if err != nil {
			t0.Fatal(err)
		}
	}
	t.network = net

	t.traceOn  = true
	t.erouter  = NewEventRouter()
	t.gotracer = NewTraceCollector(t0)
	t0.SetEventRouter(t.erouter.RouteEvent)

	t.gotracer.Attach()

	t.runCtx, t.runCancel = context.WithCancel(context.Background())
	t.runWG = xsync.NewWorkGroup(t.runCtx)

	return t
}

// Stop stops the cluster.
//
// After Stop completes all processes of the cluster are stopped.
func (t *tCluster) Stop() {
	t.Helper()
	t.TraceOff()
//	t.Done()	XXX kill - tracetest was fixed to automatically detect "extra send" deadlocks

	t.runCancel()
	err := t.runWG.Wait()
	if xcontext.Canceled(err) { // cancel is expected
		err = nil
	}
	err2 := t.network.Close()
	if err == nil {
		err = err2
	}
	if err != nil {
		t.Fatalf("run shutdown: %s", err)
	}
}

// TraceOff disables tracing.
// Trace events will stop to be created and delivered.
func (t *tCluster) TraceOff() {
	t.traceOn = false
	t.gotracer.Detach()
	//t.pytracer.Detach()

	t.tabMu.Lock()
	defer t.tabMu.Unlock()
	for _, tnode := range t.nodeTab {
		tnode.net.TraceOff()
	}
}


// registerNewNode registers new node with given name.
//
// The node is registered in .nodeTab and .erouter is updated for on-node and node-* events.
// The node must not be registered before.
func (t *tCluster) registerNewNode(name string) *tNode {
	t.tabMu.Lock()
	defer t.tabMu.Unlock()

	// check node not yet registered
	if _, already := t.nodeTab[name]; already {
		panic(fmt.Sprintf("test cluster: node %q registered twice", name))
	}

	host, err := t.network.NewHost(t.runCtx, name)
	if err != nil {
		t.Fatalf("register new node %q: %s", name, err)
	}

	// trace of events local to node
	t.erouter.BranchNode(name, name)

	// trace for events on links of all node1-node2 pairs
	for name2 := range t.nodeTab {
		// traces for events of 2 streams:
		// 1) with cause root being node1 -> node2 send, and
		// 2) ----//----            node1 <- node2 send.
		t.erouter.BranchLink(name + "-" + name2, name+"-"+name2, name2+"-"+name)
	}

	node := &tNode{}
	node.net = xnet.NetTrace(host, t.gotracer)
	if !t.traceOn {
		node.net.TraceOff()
	}

	t.nodeTab[name] = node
	return node
}

// getNode returns node by name.
//
// The node must be registered before with registerNewNode.
func (t *tCluster) getNode(name string) *tNode {
	t.tabMu.Lock()
	defer t.tabMu.Unlock()

	tnode, ok := t.nodeTab[name]
	if !ok {
		t.Fatalf("no node %q", name)
	}
	return tnode
}


// NewMaster creates new master on node name.
//
// The master will be accepting incoming connections at node:1.
// The node must be not yet existing and will be fully dedicated to the created master.
func (t *tCluster) NewMaster(name string) ITestMaster {
	tnode := t.registerNewNode(name)
	m := tNewMaster(t.name, ":1", tnode.net)
	tnode.itest = m

	// let tracer know how to map state addresses to node names
	t.gotracer.RegisterNode(m.node, name)

	// start the node
	t.runWG.Go(func(ctx context.Context) error {
		return m.Run(ctx)
	})

	return m
}

// Master returns master node that was previously created via NewMaster(name).
func (t *tCluster) Master(name string) ITestMaster {
	tnode := t.getNode(name)
	m, ok := tnode.itest.(ITestMaster)
	if !ok {
		t.Fatalf("node %q is of type %T, not master", name, tnode.itest)
	}
	return m
}


// {New,}Storage are similar to {New,}Master but for storage nodes.
func (t *tCluster) NewStorage(name, masterAddr string, back storage.Backend) ITestStorage {
	tnode := t.registerNewNode(name)
	s := tNewStorage(t.name, masterAddr, ":1", tnode.net, back)
	t.gotracer.RegisterNode(s.node.Node, name)
	t.runWG.Go(func(ctx context.Context) error {
		return s.Run(ctx)
	})
	return s
}
func (t *tCluster) Storage(name string) ITestStorage {
	tnode := t.getNode(name)
	s, ok := tnode.itest.(ITestStorage)
	if !ok {
		t.Fatalf("node %q is of type %T, not storage", name, tnode.itest)
	}
	return s
}

// {New,}Client are similar to {New,}Master but for client nodes.
func (t *tCluster) NewClient(name, masterAddr string) ITestClient {
	tnode := t.registerNewNode(name)
	c := NewClient(t.name, []string{masterAddr}, tnode.net)
	t.gotracer.RegisterNode(c.node.Node, name)
	t.runWG.Go(func(ctx context.Context) error {
		return c.Run(ctx)
	})
	return c
}
func (t *tCluster) Client(name string) ITestClient {
	tnode := t.getNode(name)
	c, ok := tnode.itest.(ITestClient)
	if !ok {
		t.Fatalf("node %q is of type %T, not client", name, tnode.itest)
	}
	return c
}


// tStorage is test-wrapper around Storage.
//
// - to automatically listen by address, not provided listener.
type tStorage struct {
	*Storage
	serveAddr string
}

func tNewStorage(clusterName, masterAddr, serveAddr string, net xnet.Networker, back storage.Backend) *tStorage {
	return &tStorage{
		Storage:   NewStorage(clusterName, []string{masterAddr}, net, back),
		serveAddr: serveAddr,
	}
}

func (s *tStorage) Run(ctx context.Context) error {
	l, err := s.node.Net.Listen(ctx, s.serveAddr)
	if err != nil {
		return err
	}

	return s.Storage.Run(ctx, l)
}

// tMaster is test-wrapper around Master.
//
// - automatically listens by address, not provided listener.
// - uses virtual clock.
type tMaster struct {
	*Master
	serveAddr string
	vclock    vclock
}

func tNewMaster(clusterName, serveAddr string, net xnet.Networker) *tMaster {
	m := &tMaster{
		Master:   NewMaster(clusterName, net),
		serveAddr: serveAddr,
	}
	m.Master.monotime = m.vclock.monotime
	return m
}

func (m *tMaster) Run(ctx context.Context) error {
	l, err := m.node.Net.Listen(ctx, m.serveAddr)
	if err != nil {
		return err
	}

	return m.Master.Run(ctx, l)
}

// ----------------------------------------

// tNewCluster_MS starts simple NEO cluster with 1 master and 1 storage nodes.
// The cluster is returned in ready-to-start state.
func (t0 *tEnv) NewCluster_MS(name string, Sback storage.Backend) *tCluster {
	t := t0.NewCluster(name)
	t.NewMaster("m")
	t.NewStorage("s", "m:1", Sback)

	// M starts listening
	t.Expect("m", netlisten("m:1"))
	t.Expect("m", δnode("m", "m:1", proto.MASTER, 1, proto.RUNNING, proto.IdTimeNone))
	t.Expect("m", clusterState("m", proto.ClusterRecovering))

	// S starts listening
	t.Expect("s", netlisten("s:1"))

	// S connects to M
	t.Expect("s-m", netdial("s", "m:1"))
	t.Expect("s-m", netconnect("s:2", "m:2",  "m:1"))
	t.Expect("s-m", conntx("s:2", "m:2", 1, &proto.RequestIdentification{
		NodeType:	proto.STORAGE,
		NID:		0,
		Address:	xnaddr("s:1"),
		ClusterName:	name,
		IdTime:		proto.IdTimeNone,
		DevPath:        nil,
		NewNID:         nil,
	}))

	t.Expect("m", δnode("m", "s:1", proto.STORAGE, 1, proto.PENDING, 0.01))

	t.Expect("s-m", conntx("m:2", "s:2", 1, &proto.AcceptIdentification{
		NodeType:	proto.MASTER,
		MyNID:		proto.NID(proto.MASTER, 1),
		YourNID:	proto.NID(proto.STORAGE, 1),
	}))

	// M sends nodeTab to S
	// XXX better notifications should go to "m-s/notify" instead ?
	t.Expect("m-s", conntx("m:2", "s:2", 0, &proto.NotifyNodeInformation{
		IdTime:		proto.IdTimeNone,	// XXX ?
		NodeList:	[]proto.NodeInfo{
			nodei("m:1", proto.MASTER,  1, proto.RUNNING, proto.IdTimeNone),
			nodei("s:1", proto.STORAGE, 1, proto.PENDING, 0.01),
		},
	}))
	t.Expect("s", δnode("s", "m:1", proto.MASTER,  1, proto.RUNNING, proto.IdTimeNone))
	t.Expect("s", δnode("s", "s:1", proto.STORAGE, 1, proto.PENDING, 0.01))

	// M starts recovery on S
	t.Expect("m-s", conntx("m:2", "s:2", 2, &proto.Recovery{}))
	t.Expect("m-s", conntx("s:2", "m:2", 2, &proto.AnswerRecovery{
		// empty new node
		PTid:		0,
		BackupTid:	proto.INVALID_TID,
		TruncateTid:	proto.INVALID_TID,
	}))

	t.Expect("m-s", conntx("m:2", "s:2", 4, &proto.AskPartitionTable{}))
	t.Expect("m-s", conntx("s:2", "m:2", 4, &proto.AnswerPartitionTable{
		PTid:		0,
		NumReplicas:	0,
		RowList:	[]proto.RowInfo{},
	}))

	// M ready to start: new cluster, no in-progress S recovery
	t.Expect("m", masterStartReady("m", true))

	return t
}
