go/zodb/fs1: Fix it to provide "Close vs watchq" guaranty

Provide guaranty that Close forces the driver to stop sending to watchq
and to close it. See a5dbb92b ("go/zodb: Require drivers to close watchq
on Close") for details.

go/zodb/storage/fs1/filestorage.go

-// Copyright (C) 2017-2020  Nexedi SA and Contributors.
+// Copyright (C) 2017-2021  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -109,6 +109,9 @@ type FileStorage struct {
 	downOnce sync.Once      // shutdown may be due to both Close and IO error in watcher
 	errClose error          // error from .file.Close()
 	watchWg  sync.WaitGroup // to wait for watcher finish
+
+	closed    chan struct{} // ready when storage was Closed
+	closeOnce sync.Once
 }
 
 // IStorageDriver
@@ -455,7 +458,7 @@ func (fs *FileStorage) watcher(w *fsnotify.Watcher, errFirstRead chan<- error) {
 		// XXX it can also be internal.poll.ErrFileClosing
 		e.Err.Error() == "use of closed file") {
 		select {
-		case <-fs.down:
+		case <-fs.closed:
 			err = nil
 		default:
 		}
@@ -472,7 +475,13 @@ func (fs *FileStorage) watcher(w *fsnotify.Watcher, errFirstRead chan<- error) {
 
 	if fs.watchq != nil {
 		if err != nil {
-			fs.watchq <- &zodb.EventError{err}
+			select {
+			case <-fs.closed:
+				// closed - skip send to watchq
+
+			case fs.watchq <- &zodb.EventError{err}:
+				// ok
+			}
 		}
 		close(fs.watchq)
 	}
@@ -522,9 +531,9 @@ mainloop:
 		if !first {
 			traceWatch("select ...")
 			select {
-			case <-fs.down:
+			case <-fs.closed:
 				// closed
-				traceWatch("down")
+				traceWatch("closed")
 				return nil
 
 			case err := <-w.Errors:
@@ -680,7 +689,7 @@ mainloop:
 			// notify client
 			if fs.watchq != nil {
 				select {
-				case <-fs.down:
+				case <-fs.closed:
 					return nil
 
 				case fs.watchq <- &zodb.EventCommit{it.Txnh.Tid, δoid}:
@@ -758,6 +767,9 @@ func (fs *FileStorage) shutdown(reason error) {
 }
 
 func (fs *FileStorage) Close() error {
+	fs.closeOnce.Do(func() {
+		close(fs.closed)
+	})
 	fs.shutdown(fmt.Errorf("closed"))
 	fs.watchWg.Wait()
 
@@ -780,6 +792,7 @@ func Open(ctx context.Context, path string, opt *zodb.DriverOptions) (_ *FileSto
 	fs := &FileStorage{
 		watchq: opt.Watchq,
 		down:   make(chan struct{}),
+		closed: make(chan struct{}),
 	}
 
 	f, err := os.Open(path)