// Copyright (C) 2017-2021  Nexedi SA and Contributors.
//                          Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.

package neo
// client node with ZODB storage interface for accessing NEO cluster.

import (
	"context"
	"fmt"
	"math/rand"
	"net/url"
	"os"
	"strings"
	"sync"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"lab.nexedi.com/kirr/go123/mem"
	"lab.nexedi.com/kirr/go123/xerr"
	"lab.nexedi.com/kirr/go123/xcontext"
	"lab.nexedi.com/kirr/go123/xnet"
	"lab.nexedi.com/kirr/go123/xsync"

	"lab.nexedi.com/kirr/neo/go/internal/task"
	taskctx "lab.nexedi.com/kirr/neo/go/internal/xcontext/task"
	"lab.nexedi.com/kirr/neo/go/internal/xurl"
	"lab.nexedi.com/kirr/neo/go/internal/xzlib"
	"lab.nexedi.com/kirr/neo/go/internal/xzodb"
	"lab.nexedi.com/kirr/neo/go/neo/internal/xsha1"
	"lab.nexedi.com/kirr/neo/go/neo/neonet"
	"lab.nexedi.com/kirr/neo/go/neo/proto"
	"lab.nexedi.com/kirr/neo/go/neo/xneo"
	"lab.nexedi.com/kirr/neo/go/zodb"
)

// Client is NEO node that talks to NEO cluster and exposes access to it via ZODB interfaces.
type Client struct {
	node *_MasteredNode

	// Run is run under:
	runWG     *xsync.WorkGroup
	runCancel func()

	// driver client <- watcher: database commits | errors.
	watchq chan<- zodb.Event
	head   zodb.Tid          // last invalidation received from server
	head0  zodb.Tid          // .head state on start of every (re)connection to master

	at0Mu          sync.Mutex
	at0            zodb.Tid            // at0 obtained when initially connecting to server
	eventq0        []*zodb.EventCommit // buffer for initial messages, until .at0 is initialized
	at0Initialized bool                // true after .at0 is initialized
	at0Ready       chan(struct{})      // ready after .at0 is initialized

	closeOnce sync.Once
	closed    chan struct{} // ready when Closed

	ownNet bool // true if Client "owns" networker and should release it on Close
}

var _ zodb.IStorageDriver = (*Client)(nil)

// NewClient creates new client node.
//
// It will connect to master @masterAddr and identify with specified cluster name.
// Use Run to actually start running the node.
func NewClient(clusterName string, masterAddrSlice []string, net xnet.Networker) *Client {
	c := &Client{
		node:     newMasteredNode(proto.CLIENT, clusterName, net, masterAddrSlice),
		at0Ready: make(chan struct{}),
		closed:   make(chan struct{}),
	}

	var runCtx context.Context
	runCtx, c.runCancel = context.WithCancel(context.Background())
	c.runWG = xsync.NewWorkGroup(runCtx)
	return c
}

// Close implements zodb.IStorageDriver.
func (c *Client) Close() (err error) {
	c.closeOnce.Do(func() {
		close(c.closed)
	})
	c.runCancel()
	err = c.runWG.Wait()
	if errors.Is(err, context.Canceled) {
		err = nil // we canceled it
	}

	// close networker if configured to do so
	if c.ownNet {
		__ := c.node.Net.Close()
		err = xerr.First(err, __)
	}
	return err
}

// Run starts client node and runs it until either ctx is canceled or master
// commands it to shutdown.
func (c *Client) Run(ctx context.Context) (err error) {
	ctx, cancel := context.WithCancel(ctx)
	c.runCancel = cancel

	c.runWG.Go(func(runCtx context.Context) error {
		ctx, cancel := xcontext.Merge/*Cancel*/(ctx, runCtx)
		defer cancel()

		return c.node.TalkMaster(ctx, func(ctx context.Context, mlink *_MasterLink) error {
			c.head0 = c.head
			wg := xsync.NewWorkGroup(ctx)

			// launch master notifications receiver
			wg.Go(func(ctx context.Context) error {
				return c.recvMaster(ctx, mlink)
			})

			// sync lastTid with master
			// TODO better change protocol for master to send us head right after accept
			// channel right after identification.
			wg.Go(func(ctx context.Context) error {
				return c.syncMaster(ctx, mlink)
			})

			return wg.Wait()
		})
	})

	return c.runWG.Wait()
}

// recvMaster receives and handles notifications from master.
func (c *Client) recvMaster(ctx context.Context, mlink *_MasterLink) (err error) {
	defer task.Running(&ctx, "rx")(&err)

	for {
		req, err := mlink.Recv1(ctx)
		if err != nil {
			return err
		}

		err = c.recvMaster1(req.Msg)
		req.Close()
		if err != nil {
			return err
		}
	}
}

// recvMaster1 handles 1 message from master.
func (c *Client) recvMaster1(msg proto.Msg) error {
	switch msg := msg.(type) {
	// <- committed txn
	case *proto.InvalidateObjects:
		return c.invalidateObjects(msg)
	default:
		return fmt.Errorf("unexpected message: %T", msg)
	}
}

// invalidateObjects is called by recvMaster1 on receiving invalidateObjects notification.
func (c *Client) invalidateObjects(msg *proto.InvalidateObjects) error {
	tid := msg.Tid

	// likely no need to verify for tid↑ because IStorage watcher does it.
	// However until .at0 is initialized we do not send events to IStorage,
	// so double check for monotonicity here as well.
	if tid <= c.head {
		return fmt.Errorf("bad invalidation from master: tid not ↑: %s -> %s", c.head, tid)
	}
	c.head = tid

	if c.watchq == nil {
		return nil
	}

	// invalidation event received and we have to send it to .watchq
	event := &zodb.EventCommit{Tid: tid, Changev: msg.OidList}

	c.at0Mu.Lock()
	defer c.at0Mu.Unlock()

	// queue initial events until .at0 is initialized after register
	// queued events will be sent to watchq by syncMaster after initializing .at0
	if !c.at0Initialized {
		c.eventq0 = append(c.eventq0, event)
		return nil
	}

	// at0 is initialized - ok to send current event if it goes > at0
	if tid > c.at0 {
		select {
		case <-c.closed:
			// closed - client does not read watchq anymore

		case c.watchq <- event:
			// ok
		}
	}
	return nil
}

// syncMaster asks M for DB head right after identification.
func (c *Client) syncMaster(ctx context.Context, mlink *_MasterLink) (err error) {
	defer task.Running(&ctx, "sync0")(&err) // TODO try to unify with Sync

	// query last_tid
	lastTxn := proto.AnswerLastTransaction{}
	err = mlink.Ask1(&proto.LastTransaction{}, &lastTxn)
	if err != nil {
		return err
	}

	if c.at0Initialized {
		if lastTxn.Tid != c.head0 {
			return fmt.Errorf("new transactions were committed while we were disconnected from master (%s -> %s)", c.head0, lastTxn.Tid)
		}
	} else {
		// since we read lastTid, in separate protocol exchange there is a
		// chance, that by the time when lastTid was read, some new transactions
		// were committed. This way lastTid will be > than some first
		// transactions received by watcher via "invalidateObjects" server
		// notification.
		//
		// filter-out first < at0 messages for this reason.
		//
		// TODO change NEO protocol so that when C connects to M, M sends it
		// current head and guarantees to send only followup invalidation
		// updates.
		c.at0Mu.Lock()
		c.at0 = lastTxn.Tid
		c.at0Initialized = true
		c.flushEventq0()
		c.at0Mu.Unlock()
		close(c.at0Ready)
	}

	return nil
}

// flushEventq0 flushes events queued in c.eventq0.
// must be called under .at0Mu
func (c *Client) flushEventq0() {
	if !c.at0Initialized {
		panic("flush, but .at0 not yet initialized")
	}
	if c.watchq != nil {
		for _, e := range c.eventq0 {
			if e.Tid > c.at0 {
				select {
				case <-c.closed:
					// closed - client does not read watchq anymore

				case c.watchq <- e:
					// ok
				}
			}
		}
	}
	c.eventq0 = nil
}


// --- user API calls ---

// Sync implements zodb.IStorageDriver.
func (c *Client) Sync(ctx context.Context) (head zodb.Tid, err error) {
	ctx = taskctx.Runningf(ctx, "%s: zsync", c.node.MyInfo.NID) // XXX mynid locking
	if glog.V(2) {
		task.TraceBegin(ctx)
		defer func() { task.TraceEnd(ctx, err) }()
	}
	defer func() {
		if err != nil {
			err = &zodb.OpError{URL: c.URL(), Op: "sync", Args: nil, Err: err}
		}
	}()

	err = c.node.WithOperational(ctx, func(mlink *neonet.NodeLink, _ *xneo.ClusterState) error {
		// XXX mlink can become down while we are making the call.
		// XXX do we want to return error or retry?
		reply := proto.AnswerLastTransaction{}
		err = mlink.Ask1(&proto.LastTransaction{}, &reply) // XXX ctx cancel
		if err != nil {
			return err // NOTE no need for ZODBErrDecode
		}
		head = reply.Tid
		return nil
	})
	return
}

// Load implements zodb.IStorageDriver.
func (c *Client) Load(ctx context.Context, xid zodb.Xid) (buf *mem.Buf, serial zodb.Tid, err error) {
	ctx = taskctx.Runningf(ctx, "%s: zload %s", c.node.MyInfo.NID, xid) // XXX mynid locking
	if glog.V(2) {
		task.TraceBegin(ctx)
		defer func() { task.TraceEnd(ctx, err) }()
	}
	defer func() {
		if err != nil {
			err = &zodb.OpError{URL: c.URL(), Op: "load", Args: xid, Err: err}
		}
	}()

	// Retrieve storages we might need to access.
	storv := make([]*xneo.PeerNode, 0, 1)
	err = c.node.WithOperational(ctx, func(mlink *neonet.NodeLink, cs *xneo.ClusterState) error {
		for _, cell := range cs.PartTab.Get(xid.Oid) {
			if cell.Readable() {
				stor := cs.NodeTab.Get(cell.NID)
				// this storage might not yet come up
				if stor != nil && stor.State == proto.RUNNING {
					storv = append(storv, stor)
				}
			}
		}
		return nil
	})
	if err != nil {
		return nil, 0, err
	}

	if len(storv) == 0 {
		// XXX add our Bugf which always forces +v (traceback) on such error print ?
		return nil, 0, errors.Errorf("internal inconsistency: cluster is operational, but no storages alive for oid %s", xid.Oid)
	}

	// TODO pick up 3 random storages and send load requests to them all,
	// getting the first who is the fastest to reply; retry from the
	// beginning if all are found to fail.
	stor := storv[rand.Intn(len(storv))]

	slink, err := stor.Dial(ctx)
	if err != nil {
		return nil, 0, err
	}
	// close accept after dialed (not to deadlock if S decides to send us
	// something).
	slink.CloseAccept() // NOTE need to close only after first real dial

	// on the wire it comes as "before", not "at"
	req := proto.GetObject{
		Oid:    xid.Oid,
		Before: xzodb.At2Before(xid.At),
		At:     proto.INVALID_TID,
	}

	resp := proto.AnswerObject{}
	err = slink.Ask1(&req, &resp)
	if err != nil {
		if e, ok := err.(*proto.Error); ok {
			err = proto.ZODBErrDecode(e)
		}
		return nil, 0, err
	}

	buf = resp.Data

	if !xsha1.Skip {
		checksum := xsha1.NEOSum(buf.Data)
		if checksum != resp.Checksum {
			return nil, 0, fmt.Errorf("data corrupt: checksum mismatch")
		}
	}

	if resp.Compression {
		buf2 := &mem.Buf{Data: nil}
		udata, err := xzlib.Decompress(buf.Data)
		buf.Release()
		if err != nil {
			return nil, 0, fmt.Errorf("data corrupt: %s", err)
		}
		buf2.Data = udata
		buf = buf2
	}

	// deletion is returned as empty data
	// https://lab.nexedi.com/nexedi/neoppod/blob/c1c26894/neo/client/app.py#L464-471
	if len(buf.Data) == 0 {
		buf.Release()
		return nil, 0, &zodb.NoDataError{Oid: xid.Oid, DeletedAt: resp.Serial}
	}

	// reply.NextSerial
	// reply.DataSerial
	return buf, resp.Serial, nil
}

// Iterate implements zodb.IStorageDriver.
func (c *Client) Iterate(ctx context.Context, tidMin, tidMax zodb.Tid) zodb.ITxnIterator {
	// see notes in ../NOTES:"On iteration"
	panic("TODO")
}


// ---- ZODB open/url support ----


func openClientByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb.IStorageDriver, _ zodb.Tid, err error) {
	// neo(s)://[credentials@]master1,master2,...,masterN/name?options
	defer task.Runningf(&ctx, "neo: open %s", u)(&err)

	var ssl bool
	switch u.Scheme {
	case "neo":  ssl = false
	case "neos": ssl = true
	default:     return nil, zodb.InvalidTid, fmt.Errorf("invalid scheme")
	}

	cred := u.User.String()
	// ca=ca.crt;cert=my.crt;key=my.key
	cred = strings.ReplaceAll(cred, ";", "&") // ; is no longer in default separators set https://github.com/golang/go/issues/25192
	x, err := xurl.ParseQuery(cred)
	if err != nil {
		return nil, zodb.InvalidTid, fmt.Errorf("credentials: %s", err)
	}
	// xpop pops k from credentials, defaulting to $NEO_<K> if envok.
	xpop := func(k string, envok bool) string {
		v, ok := x[k]
		if !ok && envok {
			v = os.Getenv("NEO_"+strings.ToUpper(k))
		}
		delete(x, k)
		return v
	}

	netcfg := neonet.Config{}
	netcfg.LoNode = xpop("lonode", false)

	if !ssl {
		if len(x) != 0 {
			return nil, zodb.InvalidTid, fmt.Errorf("credentials can be specified only with neos:// scheme")
		}
	} else {
		netcfg.CA   = xpop("ca",   true)
		netcfg.Cert = xpop("cert", true)
		netcfg.Key  = xpop("key",  true)

		if len(x) != 0 {
			return nil, zodb.InvalidTid, fmt.Errorf("invalid credentials: %v", x)
		}
	}

	name := u.Path
	name = strings.TrimPrefix(name, "/")
	if name == "" {
		return nil, zodb.InvalidTid, fmt.Errorf("cluster name not specified")
	}

	q, err := xurl.ParseQuery(u.RawQuery)
	if err != nil {
		return nil, zodb.InvalidTid, err
	}

	if len(q) != 0 {
		return nil, zodb.InvalidTid, fmt.Errorf("invalid query: %v", q)
	}


	if !opt.ReadOnly {
		return nil, zodb.InvalidTid, fmt.Errorf("TODO write mode not implemented")
	}

	net, err := neonet.Join(ctx, netcfg)
	if err != nil {
		return nil, zodb.InvalidTid, err
	}

	c := NewClient(name, strings.Split(u.Host, ","), net)
	c.ownNet = true
	c.watchq = opt.Watchq
	defer func() {
		if err != nil {
			c.Close()
		}
	}()

	// start client node serve loop
	errq := make(chan error, 1)
	go func() {
		err := c.Run(context.Background()) // NOTE not ctx
		// Client.Close cancels talkMaster which makes Run to return
		// `client: talk master(M): context canceled`
		// do not propagate this error to ZODB-level user.
		if errors.Is(err, context.Canceled) {
			err = nil
		}

		// close .watchq after serve is over
		c.at0Mu.Lock()
		defer c.at0Mu.Unlock()
		if c.watchq != nil {
			if err != nil && /* already flushed .eventq0 */c.at0Initialized {
				select {
				case <-c.closed:
					// closed - client does not read watchq anymore

				case c.watchq <- &zodb.EventError{Err: err}:
					// ok
				}
			}
			close(c.watchq)
			c.watchq = nil // prevent flushEventq0 to send to closed chan
		}

		errq <- err
	}()

	select {
	case <-ctx.Done():
		return nil, zodb.InvalidTid, ctx.Err()
	case <-errq:
		return nil, zodb.InvalidTid, err
	case <-c.at0Ready:
		return c, c.at0, nil
	}
}

// URL implements zodb.IStorageDriver.
func (c *Client) URL() string {
	// XXX options if such were given to open are discarded
	//     (but we need to be able to construct URL if Client was created via NewClient directly)

	zurl := "neo"
	if strings.Contains(c.node.Net.Network(), "+tls") {
		zurl += "s"
	}
	zurl += fmt.Sprintf(
		"://%s/%s", strings.Join(c.node.MasterAddrSlice, ","), c.node.ClusterName,
	)
	return zurl
}

func init() {
	zodb.RegisterDriver("neo",  openClientByURL)
	zodb.RegisterDriver("neos", openClientByURL)
}
