Merge branch 'master' into y/nodefs-cancel

* master:
  fuse: increase max readers for better throughput
  fuse: fix deadlock in TestCacheControl, update docs
  fs: allow extending LoopbackNode
  fuse: support the new macFUSE mount protocol
  fs: prepare for exposing loopback types
  fs: add a note about using os.File.Fd()
  fs: use loopbackNode for the root node too
  Test for RmChild deadlock.
  Fix deadlock in RmChild
  fuse: fix pollHack response to SETATTR
  fuse: try to fix mount errors on OSX
  fs: redeclare syscall.Dirent locally
  fuse: debug logging: mark nodeids as "nXXX"
  fs: split inode number tracking and kernel nodeid tracking
  fs: TestFsstress: add rename, link ; reduce runtime to 1 sec
  fs: add TestFsstress
  fs: detect lookupCount underflow

AUTHORS

@@ -16,5 +16,6 @@ Patrick Crosby <pcrosby@gmail.com>
 Paul Jolly <paul@myitcv.org.uk>
 Paul Warren <paul.warren@emc.com>
 Shayan Pooya <shayan@arista.com>
+Tommy Lindgren <tommy.lindgren@gmail.com>
 Valient Gough <vgough@pobox.com>
 Yongwoo Park <nnnlife@gmail.com>

benchmark/read_test.go

+// Copyright 2021 the Go-FUSE Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package benchmark
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"testing"
+
+	"github.com/hanwen/go-fuse/v2/internal/testutil"
+	"golang.org/x/sync/errgroup"
+)
+
+func BenchmarkGoFuseRead(b *testing.B) {
+	fs := &readFS{}
+	wd, clean := setupFs(fs, b.N)
+	defer clean()
+
+	jobs := 32
+	blockSize := 64 * 1024
+
+	cmds := make([]*exec.Cmd, jobs)
+
+	for i := 0; i < jobs; i++ {
+		cmds[i] = exec.Command("dd",
+			fmt.Sprintf("if=%s/foo.txt", wd),
+			"iflag=direct",
+			"of=/dev/null",
+			fmt.Sprintf("bs=%d", blockSize),
+			fmt.Sprintf("count=%d", b.N))
+		if testutil.VerboseTest() {
+			cmds[i].Stdout = os.Stdout
+			cmds[i].Stderr = os.Stderr
+		}
+	}
+
+	b.SetBytes(int64(jobs * blockSize))
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	var eg errgroup.Group
+
+	for i := 0; i < jobs; i++ {
+		i := i
+		eg.Go(func() error {
+			return cmds[i].Run()
+		})
+	}
+
+	if err := eg.Wait(); err != nil {
+		b.Fatalf("dd failed: %v", err)
+	}
+
+	b.StopTimer()
+}

benchmark/readfs.go

+// Copyright 2021 the Go-FUSE Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package benchmark
+
+import (
+	"context"
+	"syscall"
+	"time"
+
+	"github.com/hanwen/go-fuse/v2/fs"
+	"github.com/hanwen/go-fuse/v2/fuse"
+)
+
+const fileSize = 2 << 60
+
+// readFS is a filesystem that always and immediately returns zeros on read
+// operations. Useful when benchmarking the raw throughput with go-fuse.
+type readFS struct {
+	fs.Inode
+}
+
+var _ = (fs.NodeLookuper)((*readFS)(nil))
+
+func (n *readFS) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) {
+	sattr := fs.StableAttr{Mode: fuse.S_IFREG}
+	return n.NewInode(ctx, &readFS{}, sattr), fs.OK
+}
+
+var _ = (fs.NodeGetattrer)((*readFS)(nil))
+
+func (n *readFS) Getattr(ctx context.Context, f fs.FileHandle, out *fuse.AttrOut) syscall.Errno {
+	out.Size = fileSize
+	out.SetTimeout(time.Hour)
+	return fs.OK
+}
+
+var _ = (fs.NodeOpener)((*readFS)(nil))
+
+func (n *readFS) Open(ctx context.Context, flags uint32) (fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno) {
+	return &readFS{}, fuse.FOPEN_DIRECT_IO, fs.OK
+}
+
+var _ = (fs.FileReader)((*readFS)(nil))
+
+func (n *readFS) Read(ctx context.Context, dest []byte, offset int64) (fuse.ReadResult, syscall.Errno) {
+	return fuse.ReadResultData(dest), fs.OK
+}

benchmark/stat_test.go

@@ -129,7 +129,7 @@ func BenchmarkGoFuseStat(b *testing.B) {
 
 	threads := runtime.GOMAXPROCS(0)
 	if err := TestingBOnePass(b, threads, fileList, wd); err != nil {
-		log.Fatalf("TestingBOnePass %v8", err)
+		b.Fatalf("TestingBOnePass %v8", err)
 	}
 }
 
@@ -252,6 +252,8 @@ func BenchmarkCFuseThreadedStat(b *testing.B) {
 	f.Close()
 
 	mountPoint := testutil.TempDir()
+	defer os.RemoveAll(mountPoint)
+
 	cmd := exec.Command(wd+"/cstatfs",
 		"-o",
 		"entry_timeout=0.0,attr_timeout=0.0,ac_attr_timeout=0.0,negative_timeout=0.0",
@@ -274,6 +276,6 @@ func BenchmarkCFuseThreadedStat(b *testing.B) {
 	os.Lstat(mountPoint)
 	threads := runtime.GOMAXPROCS(0)
 	if err := TestingBOnePass(b, threads, fileList, mountPoint); err != nil {
-		log.Fatalf("TestingBOnePass %v", err)
+		b.Fatalf("TestingBOnePass %v", err)
 	}
 }

fs/bridge.go

@@ -7,7 +7,6 @@ package fs
 import (
 	"context"
 	"log"
-	"math/rand"
 	"sync"
 	"syscall"
 	"time"
@@ -61,10 +60,28 @@ type rawBridge struct {
 
 	// mu protects the following data.  Locks for inodes must be
 	// taken before rawBridge.mu
-	mu           sync.Mutex
-	nodes        map[uint64]*Inode
+	mu sync.Mutex
+
+	// stableAttrs is used to detect already-known nodes and hard links by
+	// looking at:
+	// 1) file type ......... StableAttr.Mode
+	// 2) inode number ...... StableAttr.Ino
+	// 3) generation number . StableAttr.Gen
+	stableAttrs  map[StableAttr]*Inode
 	automaticIno uint64
 
+	// The *Node ID* is an arbitrary uint64 identifier chosen by the FUSE library.
+	// It is used the identify *nodes* (files/directories/symlinks/...) in the
+	// communication between the FUSE library and the Linux kernel.
+	//
+	// The kernelNodeIds map translates between the NodeID and the corresponding
+	// go-fuse Inode object.
+	//
+	// A simple incrementing counter is used as the NodeID (see `nextNodeID`).
+	kernelNodeIds map[uint64]*Inode
+	// nextNodeID is the next free NodeID. Increment after copying the value.
+	nextNodeId uint64
+
 	files     []*fileEntry
 	freeFiles []uint32
 }
@@ -83,55 +100,26 @@ func (b *rawBridge) newInodeUnlocked(ops InodeEmbedder, id StableAttr, persisten
 		return ops.embed()
 	}
 
+	// Only the file type bits matter
+	id.Mode = id.Mode & syscall.S_IFMT
+	if id.Mode == 0 {
+		id.Mode = fuse.S_IFREG
+	}
+
 	if id.Ino == 0 {
+		// Find free inode number.
 		for {
 			id.Ino = b.automaticIno
 			b.automaticIno++
-			_, ok := b.nodes[id.Ino]
+			_, ok := b.stableAttrs[id]
 			if !ok {
 				break
 			}
 		}
 	}
 
-	// Only the file type bits matter
-	id.Mode = id.Mode & syscall.S_IFMT
-	if id.Mode == 0 {
-		id.Mode = fuse.S_IFREG
-	}
-
-	// the same node can be looked up through 2 paths in parallel, eg.
-	//
-	//	    root
-	//	    /  \
-	//	  dir1 dir2
-	//	    \  /
-	//	    file
-	//
-	// dir1.Lookup("file") and dir2.Lookup("file") are executed
-	// simultaneously.  The matching StableAttrs ensure that we return the
-	// same node.
-	var t time.Duration
-	t0 := time.Now()
-	for i := 1; true; i++ {
-		old := b.nodes[id.Ino]
-		if old == nil {
-			break
-		}
-		if old.stableAttr == id {
-			return old
-		}
-		b.mu.Unlock()
-
-		t = expSleep(t)
-		if i%5000 == 0 {
-			b.logf("blocked for %.0f seconds waiting for FORGET on i%d", time.Since(t0).Seconds(), id.Ino)
-		}
-		b.mu.Lock()
-	}
-
-	b.nodes[id.Ino] = ops.embed()
-	initInode(ops.embed(), ops, id, b, persistent)
+	initInode(ops.embed(), ops, id, b, persistent, b.nextNodeId)
+	b.nextNodeId++
 	return ops.embed()
 }
 
@@ -141,21 +129,6 @@ func (b *rawBridge) logf(format string, args ...interface{}) {
 	}
 }
 
-// expSleep sleeps for time `t` and returns an exponentially increasing value
-// for the next sleep time, capped at 1 ms.
-func expSleep(t time.Duration) time.Duration {
-	if t == 0 {
-		return time.Microsecond
-	}
-	time.Sleep(t)
-	// Next sleep is between t and 2*t
-	t += time.Duration(rand.Int63n(int64(t)))
-	if t >= time.Millisecond {
-		return time.Millisecond
-	}
-	return t
-}
-
 func (b *rawBridge) newInode(ctx context.Context, ops InodeEmbedder, id StableAttr, persistent bool) *Inode {
 	ch := b.newInodeUnlocked(ops, id, persistent)
 	if ch != ops.embed() {
@@ -169,34 +142,81 @@ func (b *rawBridge) newInode(ctx context.Context, ops InodeEmbedder, id StableAt
 }
 
 // addNewChild inserts the child into the tree. Returns file handle if file != nil.
-func (b *rawBridge) addNewChild(parent *Inode, name string, child *Inode, file FileHandle, fileFlags uint32, out *fuse.EntryOut) uint32 {
+// Unless fileFlags has the syscall.O_EXCL bit set, child.stableAttr will be used
+// to find an already-known node. If one is found, `child` is ignored and the
+// already-known one is used. The node that was actually used is returned.
+func (b *rawBridge) addNewChild(parent *Inode, name string, child *Inode, file FileHandle, fileFlags uint32, out *fuse.EntryOut) (selected *Inode, fh uint32) {
 	if name == "." || name == ".." {
 		log.Panicf("BUG: tried to add virtual entry %q to the actual tree", name)
 	}
-	lockNodes(parent, child)
-	parent.setEntry(name, child)
-	b.mu.Lock()
 
-	// Due to concurrent FORGETs, lookupCount may have dropped to zero.
-	// This means it MAY have been deleted from nodes[] already. Add it back.
-	if child.lookupCount == 0 {
-		b.nodes[child.stableAttr.Ino] = child
+	// the same node can be looked up through 2 paths in parallel, eg.
+	//
+	//	    root
+	//	    /  \
+	//	  dir1 dir2
+	//	    \  /
+	//	    file
+	//
+	// dir1.Lookup("file") and dir2.Lookup("file") are executed
+	// simultaneously.  The matching StableAttrs ensure that we return the
+	// same node.
+	orig := child
+	id := child.stableAttr
+	if id.Mode & ^(uint32(syscall.S_IFMT)) != 0 {
+		log.Panicf("%#v", id)
 	}
+	for {
+		lockNodes(parent, child)
+		b.mu.Lock()
+		if fileFlags&syscall.O_EXCL != 0 {
+			// must create a new node - don't look for existing nodes
+			break
+		}
+		old := b.stableAttrs[id]
+		if old == nil {
+			if child == orig {
+				// no pre-existing node under this inode number
+				break
+			} else {
+				// old inode disappeared while we were looping here. Go back to
+				// original child.
+				b.mu.Unlock()
+				unlockNodes(parent, child)
+				child = orig
+				continue
+			}
+		}
+		if old == child {
+			// we now have the right inode locked
+			break
+		}
+		// found a different existing node
+		b.mu.Unlock()
+		unlockNodes(parent, child)
+		child = old
+	}
+
 	child.lookupCount++
 	child.changeCounter++
 
-	var fh uint32
+	b.kernelNodeIds[child.nodeId] = child
+	// Any node that might be there is overwritten - it is obsolete now
+	b.stableAttrs[id] = child
 	if file != nil {
 		fh = b.registerFile(child, file, fileFlags)
 	}
 
-	out.NodeId = child.stableAttr.Ino
+	parent.setEntry(name, child)
+
+	out.NodeId = child.nodeId
 	out.Generation = child.stableAttr.Gen
 	out.Attr.Ino = child.stableAttr.Ino
 
 	b.mu.Unlock()
 	unlockNodes(parent, child)
-	return fh
+
+	return child, fh
 }
 
 func (b *rawBridge) setEntryOutTimeout(out *fuse.EntryOut) {
@@ -237,6 +257,8 @@ func NewNodeFS(root InodeEmbedder, opts *Options) fuse.RawFileSystem {
 	bridge := &rawBridge{
 		automaticIno: opts.FirstAutomaticIno,
 		server:       opts.ServerCallbacks,
+		nextNodeId:   2, // the root node has nodeid 1
+		stableAttrs:  make(map[StableAttr]*Inode),
 	}
 	if bridge.automaticIno == 1 {
 		bridge.automaticIno++
@@ -256,15 +278,16 @@ func NewNodeFS(root InodeEmbedder, opts *Options) fuse.RawFileSystem {
 
 	initInode(root.embed(), root,
 		StableAttr{
-			Ino:  1,
+			Ino:  root.embed().StableAttr().Ino,
 			Mode: fuse.S_IFDIR,
 		},
 		bridge,
 		false,
+		1,
 	)
 	bridge.root = root.embed()
 	bridge.root.lookupCount = 1
-	bridge.nodes = map[uint64]*Inode{
+	bridge.kernelNodeIds = map[uint64]*Inode{
 		1: bridge.root,
 	}
 
@@ -287,7 +310,7 @@ func (b *rawBridge) String() string {
 func (b *rawBridge) inode(id uint64, fh uint64) (*Inode, *fileEntry) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
-	n, f := b.nodes[id], b.files[fh]
+	n, f := b.kernelNodeIds[id], b.files[fh]
 	if n == nil {
 		log.Panicf("unknown node %d", id)
 	}
@@ -306,8 +329,8 @@ func (b *rawBridge) Lookup(cancel <-chan struct{}, header *fuse.InHeader, name s
 		return errnoToStatus(errno)
 	}
 
+	child, _ = b.addNewChild(parent, name, child, nil, 0, out)
 	child.setEntryOut(out)
-	b.addNewChild(parent, name, child, nil, 0, out)
 	b.setEntryOutTimeout(out)
 	return fuse.OK
 }
@@ -382,8 +405,8 @@ func (b *rawBridge) Mkdir(cancel <-chan struct{}, input *fuse.MkdirIn, name stri
 		log.Panicf("Mkdir: mode must be S_IFDIR (%o), got %o", fuse.S_IFDIR, out.Attr.Mode)
 	}
 
+	child, _ = b.addNewChild(parent, name, child, nil, syscall.O_EXCL, out)
 	child.setEntryOut(out)
-	b.addNewChild(parent, name, child, nil, 0, out)
 	b.setEntryOutTimeout(out)
 	return fuse.OK
 }
@@ -401,8 +424,8 @@ func (b *rawBridge) Mknod(cancel <-chan struct{}, input *fuse.MknodIn, name stri
 		return errnoToStatus(errno)
 	}
 
+	child, _ = b.addNewChild(parent, name, child, nil, syscall.O_EXCL, out)
 	child.setEntryOut(out)
-	b.addNewChild(parent, name, child, nil, 0, out)
 	b.setEntryOutTimeout(out)
 	return fuse.OK
 }
@@ -428,8 +451,9 @@ func (b *rawBridge) Create(cancel <-chan struct{}, input *fuse.CreateIn, name st
 		return errnoToStatus(errno)
 	}
 
-	out.Fh = uint64(b.addNewChild(parent, name, child, f, input.Flags|syscall.O_CREAT, &out.EntryOut))
+	child, fh := b.addNewChild(parent, name, child, f, input.Flags|syscall.O_CREAT|syscall.O_EXCL, &out.EntryOut)
 
+	out.Fh = uint64(fh)
 	out.OpenFlags = flags
 
 	child.setEntryOut(&out.EntryOut)
@@ -540,8 +564,8 @@ func (b *rawBridge) Link(cancel <-chan struct{}, input *fuse.LinkIn, name string
 			return errnoToStatus(errno)
 		}
 
+		child, _ = b.addNewChild(parent, name, child, nil, 0, out)
 		child.setEntryOut(out)
-		b.addNewChild(parent, name, child, nil, 0, out)
 		b.setEntryOutTimeout(out)
 		return fuse.OK
 	}
@@ -557,7 +581,7 @@ func (b *rawBridge) Symlink(cancel <-chan struct{}, header *fuse.InHeader, targe
 			return errnoToStatus(status)
 		}
 
-		b.addNewChild(parent, name, child, nil, 0, out)
+		child, _ = b.addNewChild(parent, name, child, nil, syscall.O_EXCL, out)
 		child.setEntryOut(out)
 		b.setEntryOutTimeout(out)
 		return fuse.OK
@@ -769,7 +793,7 @@ func (b *rawBridge) releaseFileEntry(nid uint64, fh uint64) (*Inode, *fileEntry)
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
-	n := b.nodes[nid]
+	n := b.kernelNodeIds[nid]
 	var entry *fileEntry
 	if fh > 0 {
 		last := len(n.openFiles) - 1
@@ -994,7 +1018,7 @@ func (b *rawBridge) ReadDirPlus(cancel <-chan struct{}, input *fuse.ReadIn, out
 				entryOut.SetEntryTimeout(*b.options.NegativeTimeout)
 			}
 		} else {
-			b.addNewChild(n, e.Name, child, nil, 0, entryOut)
+			child, _ = b.addNewChild(n, e.Name, child, nil, 0, entryOut)
 			child.setEntryOut(entryOut)
 			b.setEntryOutTimeout(entryOut)
 			if e.Mode&syscall.S_IFMT != child.stableAttr.Mode&syscall.S_IFMT {

fs/dirstream_linux.go

@@ -55,12 +55,26 @@ func (ds *loopbackDirStream) HasNext() bool {
 	return len(ds.todo) > 0
 }
 
+// Like syscall.Dirent, but without the [256]byte name.
+type dirent struct {
+	Ino    uint64
+	Off    int64
+	Reclen uint16
+	Type   uint8
+	Name   [1]uint8 // align to 4 bytes for 32 bits.
+}
+
 func (ds *loopbackDirStream) Next() (fuse.DirEntry, syscall.Errno) {
 	ds.mu.Lock()
 	defer ds.mu.Unlock()
-	de := (*syscall.Dirent)(unsafe.Pointer(&ds.todo[0]))
 
-	nameBytes := ds.todo[unsafe.Offsetof(syscall.Dirent{}.Name):de.Reclen]
+	// We can't use syscall.Dirent here, because it declares a
+	// [256]byte name, which may run beyond the end of ds.todo.
+	// when that happens in the race detector, it causes a panic
+	// "converted pointer straddles multiple allocations"
+	de := (*dirent)(unsafe.Pointer(&ds.todo[0]))
+
+	nameBytes := ds.todo[unsafe.Offsetof(dirent{}.Name):de.Reclen]
 	ds.todo = ds.todo[de.Reclen:]
 
 	// After the loop, l contains the index of the first '\0'.

fs/files.go

@@ -17,7 +17,9 @@ import (
 )
 
 // NewLoopbackFile creates a FileHandle out of a file descriptor. All
-// operations are implemented.
+// operations are implemented. When using the Fd from a *os.File, call
+// syscall.Dup() on the fd, to avoid os.File's finalizer from closing
+// the file descriptor.
 func NewLoopbackFile(fd int) FileHandle {
 	return &loopbackFile{fd: fd}
 }

fs/forget_test.go

@@ -112,7 +112,7 @@ func TestForget(t *testing.T) {
 
 	bridge := rawFS.(*rawBridge)
 	bridge.mu.Lock()
-	l := len(bridge.nodes)
+	l := len(bridge.kernelNodeIds)
 	bridge.mu.Unlock()
 	if l != 1 {
 		t.Fatalf("got %d live nodes, want 1", l)

fs/inode.go

@@ -63,6 +63,11 @@ type Inode struct {
 	ops    InodeEmbedder
 	bridge *rawBridge
 
+	// The *Node ID* is an arbitrary uint64 identifier chosen by the FUSE library.
+	// It is used the identify *nodes* (files/directories/symlinks/...) in the
+	// communication between the FUSE library and the Linux kernel.
+	nodeId uint64
+
 	// Following data is mutable.
 
 	// file handles.
@@ -115,11 +120,12 @@ func (n *Inode) EmbeddedInode() *Inode {
 	return n
 }
 
-func initInode(n *Inode, ops InodeEmbedder, attr StableAttr, bridge *rawBridge, persistent bool) {
+func initInode(n *Inode, ops InodeEmbedder, attr StableAttr, bridge *rawBridge, persistent bool, nodeId uint64) {
 	n.ops = ops
 	n.stableAttr = attr
 	n.bridge = bridge
 	n.persistent = persistent
+	n.nodeId = nodeId
 	n.parents = make(map[parentData]struct{})
 	if attr.Mode == fuse.S_IFDIR {
 		n.children = make(map[string]*Inode)
@@ -128,7 +134,7 @@ func initInode(n *Inode, ops InodeEmbedder, attr StableAttr, bridge *rawBridge,
 
 // Set node ID and mode in EntryOut
 func (n *Inode) setEntryOut(out *fuse.EntryOut) {
-	out.NodeId = n.stableAttr.Ino
+	out.NodeId = n.nodeId
 	out.Ino = n.stableAttr.Ino
 	out.Mode = (out.Attr.Mode & 07777) | n.stableAttr.Mode
 }
@@ -303,8 +309,8 @@ func (n *Inode) Path(root *Inode) string {
 
 	if root != nil && root != p {
 		deletedPlaceholder := fmt.Sprintf(".go-fuse.%d/deleted", rand.Uint64())
-		n.bridge.logf("warning: Inode.Path: inode i%d is orphaned, replacing segment with %q",
-			n.stableAttr.Ino, deletedPlaceholder)
+		n.bridge.logf("warning: Inode.Path: n%d is orphaned, replacing segment with %q",
+			n.nodeId, deletedPlaceholder)
 		// NOSUBMIT - should replace rather than append?
 		segments = append(segments, deletedPlaceholder)
 	}
@@ -329,7 +335,16 @@ func (n *Inode) Path(root *Inode) string {
 // but it could be also valid if only iparent is locked and ichild was just
 // created and only one goroutine keeps referencing it.
 func (iparent *Inode) setEntry(name string, ichild *Inode) {
-	ichild.parents[parentData{name, iparent}] = struct{}{}
+	newParent := parentData{name, iparent}
+	if ichild.stableAttr.Mode == syscall.S_IFDIR {
+		// Directories cannot have more than one parent. Clear the map.
+		// This special-case is neccessary because ichild may still have a
+		// parent that was forgotten (i.e. removed from bridge.inoMap).
+		for i := range ichild.parents {
+			delete(ichild.parents, i)
+		}
+	}
+	ichild.parents[newParent] = struct{}{}
 	iparent.children[name] = ichild
 	ichild.changeCounter++
 	iparent.changeCounter++
@@ -360,8 +375,8 @@ func (n *Inode) ForgetPersistent() {
 // NewInode returns an inode for the given InodeEmbedder. The mode
 // should be standard mode argument (eg. S_IFDIR). The inode number in
 // id.Ino argument is used to implement hard-links.  If it is given,
-// and another node with the same ID is known, that will node will be
-// returned, and the passed-in `node` is ignored.
+// and another node with the same ID is known, the new inode may be
+// ignored, and the old one used instead.
 func (n *Inode) NewInode(ctx context.Context, node InodeEmbedder, id StableAttr) *Inode {
 	return n.newInode(ctx, node, id, false)
 }
@@ -380,8 +395,9 @@ func (n *Inode) removeRef(nlookup uint64, dropPersistence bool) (forgotten bool,
 	n.mu.Lock()
 	if nlookup > 0 && dropPersistence {
 		log.Panic("only one allowed")
+	} else if nlookup > n.lookupCount {
+		log.Panicf("n%d lookupCount underflow: lookupCount=%d, decrement=%d", n.nodeId, n.lookupCount, nlookup)
 	} else if nlookup > 0 {
-
 		n.lookupCount -= nlookup
 		n.changeCounter++
 	} else if dropPersistence && n.persistent {
@@ -389,13 +405,22 @@ func (n *Inode) removeRef(nlookup uint64, dropPersistence bool) (forgotten bool,
 		n.changeCounter++
 	}
 
+	n.bridge.mu.Lock()
+	if n.lookupCount == 0 {
+		forgotten = true
+		// Dropping the node from inoMap guarantees that no new references to this node are
+		// handed out to the kernel, hence we can also safely delete it from nodeidMap.
+		delete(n.bridge.stableAttrs, n.stableAttr)
+		delete(n.bridge.kernelNodeIds, n.nodeId)
+	}
+	n.bridge.mu.Unlock()
+
 retry:
 	for {
 		lockme = append(lockme[:0], n)
 		parents = parents[:0]
 		nChange := n.changeCounter
 		live = n.lookupCount > 0 || len(n.children) > 0 || n.persistent
-		forgotten = n.lookupCount == 0
 		for p := range n.parents {
 			parents = append(parents, p)
 			lockme = append(lockme, p.parent)
@@ -415,6 +440,10 @@ retry:
 		}
 
 		for _, p := range parents {
+			if p.parent.children[p.name] != n {
+				// another node has replaced us already
+				continue
+			}
 			delete(p.parent.children, p.name)
 			p.parent.changeCounter++
 		}
@@ -422,13 +451,9 @@ retry:
 		n.changeCounter++
 
 		if n.lookupCount != 0 {
-			panic("lookupCount changed")
+			log.Panicf("n%d %p lookupCount changed: %d", n.nodeId, n, n.lookupCount)
 		}
 
-		n.bridge.mu.Lock()
-		delete(n.bridge.nodes, n.stableAttr.Ino)
-		n.bridge.mu.Unlock()
-
 		unlockNodes(lockme...)
 		break
 	}
@@ -558,8 +583,6 @@ retry:
 		lockNodes(lockme...)
 		if n.changeCounter != nChange {
 			unlockNodes(lockme...)
-			// could avoid unlocking and relocking n here.
-			n.mu.Lock()
 			continue retry
 		}
 
@@ -712,7 +735,7 @@ retry:
 // tuple should be invalidated. On next access, a LOOKUP operation
 // will be started.
 func (n *Inode) NotifyEntry(name string) syscall.Errno {
-	status := n.bridge.server.EntryNotify(n.stableAttr.Ino, name)
+	status := n.bridge.server.EntryNotify(n.nodeId, name)
 	return syscall.Errno(status)
 }
 
@@ -721,7 +744,7 @@ func (n *Inode) NotifyEntry(name string) syscall.Errno {
 // to NotifyEntry, but also sends an event to inotify watchers.
 func (n *Inode) NotifyDelete(name string, child *Inode) syscall.Errno {
 	// XXX arg ordering?
-	return syscall.Errno(n.bridge.server.DeleteNotify(n.stableAttr.Ino, child.stableAttr.Ino, name))
+	return syscall.Errno(n.bridge.server.DeleteNotify(n.nodeId, child.nodeId, name))
 
 }
 
@@ -729,16 +752,16 @@ func (n *Inode) NotifyDelete(name string, child *Inode) syscall.Errno {
 // inode should be flushed from buffers.
 func (n *Inode) NotifyContent(off, sz int64) syscall.Errno {
 	// XXX how does this work for directories?
-	return syscall.Errno(n.bridge.server.InodeNotify(n.stableAttr.Ino, off, sz))
+	return syscall.Errno(n.bridge.server.InodeNotify(n.nodeId, off, sz))
 }
 
 // WriteCache stores data in the kernel cache.
 func (n *Inode) WriteCache(offset int64, data []byte) syscall.Errno {
-	return syscall.Errno(n.bridge.server.InodeNotifyStoreCache(n.stableAttr.Ino, offset, data))
+	return syscall.Errno(n.bridge.server.InodeNotifyStoreCache(n.nodeId, offset, data))
 }
 
 // ReadCache reads data from the kernel cache.
 func (n *Inode) ReadCache(offset int64, dest []byte) (count int, errno syscall.Errno) {
-	c, s := n.bridge.server.InodeRetrieveCache(n.stableAttr.Ino, offset, dest)
+	c, s := n.bridge.server.InodeRetrieveCache(n.nodeId, offset, dest)
 	return c, syscall.Errno(s)
 }

fs/loopback_darwin.go

@@ -16,23 +16,23 @@ import (
 	"github.com/hanwen/go-fuse/v2/internal/utimens"
 )
 
-func (n *loopbackNode) Getxattr(ctx context.Context, attr string, dest []byte) (uint32, syscall.Errno) {
+func (n *LoopbackNode) Getxattr(ctx context.Context, attr string, dest []byte) (uint32, syscall.Errno) {
 	return 0, syscall.ENOSYS
 }
 
-func (n *loopbackNode) Setxattr(ctx context.Context, attr string, data []byte, flags uint32) syscall.Errno {
+func (n *LoopbackNode) Setxattr(ctx context.Context, attr string, data []byte, flags uint32) syscall.Errno {
 	return syscall.ENOSYS
 }
 
-func (n *loopbackNode) Removexattr(ctx context.Context, attr string) syscall.Errno {
+func (n *LoopbackNode) Removexattr(ctx context.Context, attr string) syscall.Errno {
 	return syscall.ENOSYS
 }
 
-func (n *loopbackNode) Listxattr(ctx context.Context, dest []byte) (uint32, syscall.Errno) {
+func (n *LoopbackNode) Listxattr(ctx context.Context, dest []byte) (uint32, syscall.Errno) {
 	return 0, syscall.ENOSYS
 }
 
-func (n *loopbackNode) renameExchange(name string, newparent *loopbackNode, newName string) syscall.Errno {
+func (n *LoopbackNode) renameExchange(name string, newparent InodeEmbedder, newName string) syscall.Errno {
 	return syscall.ENOSYS
 }
 
@@ -111,7 +111,7 @@ func (f *loopbackFile) utimens(a *time.Time, m *time.Time) syscall.Errno {
 	return ToErrno(err)
 }
 
-func (n *loopbackNode) CopyFileRange(ctx context.Context, fhIn FileHandle,
+func (n *LoopbackNode) CopyFileRange(ctx context.Context, fhIn FileHandle,
 	offIn uint64, out *Inode, fhOut FileHandle, offOut uint64,
 	len uint64, flags uint64) (uint32, syscall.Errno) {
 	return 0, syscall.ENOSYS

fs/loopback_linux.go

@@ -8,38 +8,40 @@ package fs
 
 import (
 	"context"
+	"path/filepath"
 	"syscall"
 
 	"golang.org/x/sys/unix"
 )
 
-func (n *loopbackNode) Getxattr(ctx context.Context, attr string, dest []byte) (uint32, syscall.Errno) {
+func (n *LoopbackNode) Getxattr(ctx context.Context, attr string, dest []byte) (uint32, syscall.Errno) {
 	sz, err := unix.Lgetxattr(n.path(), attr, dest)
 	return uint32(sz), ToErrno(err)
 }
 
-func (n *loopbackNode) Setxattr(ctx context.Context, attr string, data []byte, flags uint32) syscall.Errno {
+func (n *LoopbackNode) Setxattr(ctx context.Context, attr string, data []byte, flags uint32) syscall.Errno {
 	err := unix.Lsetxattr(n.path(), attr, data, int(flags))
 	return ToErrno(err)
 }
 
-func (n *loopbackNode) Removexattr(ctx context.Context, attr string) syscall.Errno {
+func (n *LoopbackNode) Removexattr(ctx context.Context, attr string) syscall.Errno {
 	err := unix.Lremovexattr(n.path(), attr)
 	return ToErrno(err)
 }
 
-func (n *loopbackNode) Listxattr(ctx context.Context, dest []byte) (uint32, syscall.Errno) {
+func (n *LoopbackNode) Listxattr(ctx context.Context, dest []byte) (uint32, syscall.Errno) {
 	sz, err := unix.Llistxattr(n.path(), dest)
 	return uint32(sz), ToErrno(err)
 }
 
-func (n *loopbackNode) renameExchange(name string, newparent *loopbackNode, newName string) syscall.Errno {
+func (n *LoopbackNode) renameExchange(name string, newparent InodeEmbedder, newName string) syscall.Errno {
 	fd1, err := syscall.Open(n.path(), syscall.O_DIRECTORY, 0)
 	if err != nil {
 		return ToErrno(err)
 	}
 	defer syscall.Close(fd1)
-	fd2, err := syscall.Open(newparent.path(), syscall.O_DIRECTORY, 0)
+	p2 := filepath.Join(n.RootData.Path, newparent.EmbeddedInode().Path(nil))
+	fd2, err := syscall.Open(p2, syscall.O_DIRECTORY, 0)
 	defer syscall.Close(fd2)
 	if err != nil {
 		return ToErrno(err)
@@ -52,22 +54,22 @@ func (n *loopbackNode) renameExchange(name string, newparent *loopbackNode, newN
 
 	// Double check that nodes didn't change from under us.
 	inode := &n.Inode
-	if inode.Root() != inode && inode.StableAttr().Ino != n.root().idFromStat(&st).Ino {
+	if inode.Root() != inode && inode.StableAttr().Ino != n.RootData.idFromStat(&st).Ino {
 		return syscall.EBUSY
 	}
 	if err := syscall.Fstat(fd2, &st); err != nil {
 		return ToErrno(err)
 	}
 
-	newinode := &newparent.Inode
-	if newinode.Root() != newinode && newinode.StableAttr().Ino != n.root().idFromStat(&st).Ino {
+	newinode := newparent.EmbeddedInode()
+	if newinode.Root() != newinode && newinode.StableAttr().Ino != n.RootData.idFromStat(&st).Ino {
 		return syscall.EBUSY
 	}
 
 	return ToErrno(unix.Renameat2(fd1, name, fd2, newName, unix.RENAME_EXCHANGE))
 }
 
-func (n *loopbackNode) CopyFileRange(ctx context.Context, fhIn FileHandle,
+func (n *LoopbackNode) CopyFileRange(ctx context.Context, fhIn FileHandle,
 	offIn uint64, out *Inode, fhOut FileHandle, offOut uint64,
 	len uint64, flags uint64) (uint32, syscall.Errno) {
 	lfIn, ok := fhIn.(*loopbackFile)

fs/rmchild_test.go

+// Copyright 2019 the Go-FUSE Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fs
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/hanwen/go-fuse/v2/fuse"
+)
+
+func TestRmChildParallel(t *testing.T) {
+	want := "hello"
+	root := &Inode{}
+	_, _, clean := testMount(t, root, &Options{
+		FirstAutomaticIno: 1,
+		OnAdd: func(ctx context.Context) {
+			n := root.EmbeddedInode()
+
+			var wg sync.WaitGroup
+			var nms []string
+			for i := 0; i < 100; i++ {
+				nms = append(nms, fmt.Sprint(i))
+			}
+			for _, nm := range nms {
+				wg.Add(1)
+				go func(nm string) {
+					ch := n.NewPersistentInode(
+						ctx,
+						&MemRegularFile{
+							Data: []byte(want),
+							Attr: fuse.Attr{
+								Mode: 0464,
+							},
+						},
+						StableAttr{})
+					n.AddChild(nm, ch, false)
+					wg.Done()
+				}(nm)
+			}
+			for _, nm := range nms {
+				wg.Add(1)
+				go func(nm string) {
+					n.RmChild(nm)
+					wg.Done()
+				}(nm)
+			}
+
+			wg.Wait()
+		},
+	})
+	defer clean()
+}

fs/simple_test.go

@@ -5,12 +5,19 @@
 package fs
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"reflect"
+	"runtime"
+	"sort"
+	"strings"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"testing"
 	"time"
@@ -392,6 +399,219 @@ func TestOpenDirectIO(t *testing.T) {
 	posixtest.DirectIO(t, tc.mntDir)
 }
 
+// TestFsstress is loosely modeled after xfstest's fsstress. It performs rapid
+// parallel removes / creates / readdirs. Coupled with inode reuse, this test
+// used to deadlock go-fuse quite quickly.
+//
+// Note: Run as
+//
+//     TMPDIR=/var/tmp go test -run TestFsstress
+//
+// to make sure the backing filesystem is ext4. /tmp is tmpfs on modern Linux
+// distributions, and tmpfs does not reuse inode numbers, hiding the problem.
+func TestFsstress(t *testing.T) {
+	tc := newTestCase(t, &testOptions{suppressDebug: true, attrCache: true, entryCache: true})
+	defer tc.Clean()
+
+	{
+		old := runtime.GOMAXPROCS(100)
+		defer runtime.GOMAXPROCS(old)
+	}
+
+	const concurrency = 10
+	var wg sync.WaitGroup
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// operations taking 1 path argument
+	ops1 := map[string]func(string) error{
+		"mkdir":      func(p string) error { return syscall.Mkdir(p, 0700) },
+		"rmdir":      func(p string) error { return syscall.Rmdir(p) },
+		"mknod_reg":  func(p string) error { return syscall.Mknod(p, 0700|syscall.S_IFREG, 0) },
+		"remove":     os.Remove,
+		"unlink":     syscall.Unlink,
+		"mknod_sock": func(p string) error { return syscall.Mknod(p, 0700|syscall.S_IFSOCK, 0) },
+		"mknod_fifo": func(p string) error { return syscall.Mknod(p, 0700|syscall.S_IFIFO, 0) },
+		"mkfifo":     func(p string) error { return syscall.Mkfifo(p, 0700) },
+		"symlink":    func(p string) error { return syscall.Symlink("foo", p) },
+		"creat": func(p string) error {
+			fd, err := syscall.Open(p, syscall.O_CREAT|syscall.O_EXCL, 0700)
+			if err == nil {
+				syscall.Close(fd)
+			}
+			return err
+		},
+	}
+	// operations taking 2 path arguments
+	ops2 := map[string]func(string, string) error{
+		"rename": syscall.Rename,
+		"link":   syscall.Link,
+	}
+
+	type opStats struct {
+		ok   *int64
+		fail *int64
+		hung *int64
+	}
+	stats := make(map[string]opStats)
+
+	// pathN() returns something like /var/tmp/TestFsstress/TestFsstress.4
+	pathN := func(n int) string {
+		return fmt.Sprintf("%s/%s.%d", tc.mntDir, t.Name(), n)
+	}
+
+	opLoop := func(k string, n int) {
+		defer wg.Done()
+		op := ops1[k]
+		for {
+			p := pathN(1)
+			atomic.AddInt64(stats[k].hung, 1)
+			err := op(p)
+			atomic.AddInt64(stats[k].hung, -1)
+			if err != nil {
+				atomic.AddInt64(stats[k].fail, 1)
+			} else {
+				atomic.AddInt64(stats[k].ok, 1)
+			}
+			select {
+			case <-ctx.Done():
+				return
+			default:
+			}
+		}
+	}
+
+	op2Loop := func(k string, n int) {
+		defer wg.Done()
+		op := ops2[k]
+		n2 := (n + 1) % concurrency
+		for {
+			p1 := pathN(n)
+			p2 := pathN(n2)
+			atomic.AddInt64(stats[k].hung, 1)
+			err := op(p1, p2)
+			atomic.AddInt64(stats[k].hung, -1)
+			if err != nil {
+				atomic.AddInt64(stats[k].fail, 1)
+			} else {
+				atomic.AddInt64(stats[k].ok, 1)
+			}
+			select {
+			case <-ctx.Done():
+				return
+			default:
+			}
+		}
+	}
+
+	readdirLoop := func(k string) {
+		defer wg.Done()
+		for {
+			atomic.AddInt64(stats[k].hung, 1)
+			f, err := os.Open(tc.mntDir)
+			if err != nil {
+				panic(err)
+			}
+			_, err = f.Readdir(0)
+			if err != nil {
+				atomic.AddInt64(stats[k].fail, 1)
+			} else {
+				atomic.AddInt64(stats[k].ok, 1)
+			}
+			f.Close()
+			atomic.AddInt64(stats[k].hung, -1)
+			select {
+			case <-ctx.Done():
+				return
+			default:
+			}
+		}
+	}
+
+	// prepare stats map
+	var allOps []string
+	for k := range ops1 {
+		allOps = append(allOps, k)
+	}
+	for k := range ops2 {
+		allOps = append(allOps, k)
+	}
+	allOps = append(allOps, "readdir")
+	for _, k := range allOps {
+		var i1, i2, i3 int64
+		stats[k] = opStats{ok: &i1, fail: &i2, hung: &i3}
+	}
+
+	// spawn worker goroutines
+	for i := 0; i < concurrency; i++ {
+		for k := range ops1 {
+			wg.Add(1)
+			go opLoop(k, i)
+		}
+		for k := range ops2 {
+			wg.Add(1)
+			go op2Loop(k, i)
+		}
+	}
+	{
+		k := "readdir"
+		wg.Add(1)
+		go readdirLoop(k)
+	}
+
+	// spawn ls loop
+	//
+	// An external "ls" loop has a destructive effect that I am unable to
+	// reproduce through in-process operations.
+	if strings.ContainsAny(tc.mntDir, "'\\") {
+		// But let's not enable shell injection.
+		log.Panicf("shell injection attempt? mntDir=%q", tc.mntDir)
+	}
+	// --color=always enables xattr lookups for extra stress
+	cmd := exec.Command("bash", "-c", "while true ; do ls -l --color=always '"+tc.mntDir+"'; done")
+	err := cmd.Start()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cmd.Process.Kill()
+
+	// Run the test for 1 second. If it deadlocks, it usually does within 20ms.
+	time.Sleep(1 * time.Second)
+
+	cancel()
+
+	// waitTimeout waits for the waitgroup for the specified max timeout.
+	// Returns true if waiting timed out.
+	waitTimeout := func(wg *sync.WaitGroup, timeout time.Duration) bool {
+		c := make(chan struct{})
+		go func() {
+			defer close(c)
+			wg.Wait()
+		}()
+		select {
+		case <-c:
+			return false // completed normally
+		case <-time.After(timeout):
+			return true // timed out
+		}
+	}
+
+	if waitTimeout(&wg, time.Second) {
+		t.Errorf("timeout waiting for goroutines to exit (deadlocked?)")
+	}
+
+	// Print operation statistics
+	var keys []string
+	for k := range stats {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	t.Logf("Operation statistics:")
+	for _, k := range keys {
+		v := stats[k]
+		t.Logf("%10s: %5d ok, %6d fail, %2d hung", k, *v.ok, *v.fail, *v.hung)
+	}
+}
+
 func init() {
 	syscall.Umask(0)
 }

fs/windows_example_test.go

+// Copyright 2020 the Go-FUSE Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fs_test
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/hanwen/go-fuse/v2/fs"
+	"github.com/hanwen/go-fuse/v2/fuse"
+)
+
+// WindowsNode emulates Windows FS semantics, which forbids deleting open files.
+type WindowsNode struct {
+	// WindowsNode inherits most functionality from LoopbackNode.
+	fs.LoopbackNode
+
+	mu        sync.Mutex
+	openCount int
+}
+
+var _ = (fs.NodeOpener)((*WindowsNode)(nil))
+
+func (n *WindowsNode) Open(ctx context.Context, flags uint32) (fs.FileHandle, uint32, syscall.Errno) {
+	fh, flags, errno := n.LoopbackNode.Open(ctx, flags)
+	if errno == 0 {
+		n.mu.Lock()
+		defer n.mu.Unlock()
+
+		n.openCount++
+	}
+	return fh, flags, errno
+}
+
+var _ = (fs.NodeCreater)((*WindowsNode)(nil))
+
+func (n *WindowsNode) Create(ctx context.Context, name string, flags uint32, mode uint32, out *fuse.EntryOut) (*fs.Inode, fs.FileHandle, uint32, syscall.Errno) {
+	inode, fh, flags, errno := n.LoopbackNode.Create(ctx, name, flags, mode, out)
+	if errno == 0 {
+		wn := inode.Operations().(*WindowsNode)
+		wn.openCount++
+	}
+
+	return inode, fh, flags, errno
+}
+
+var _ = (fs.NodeReleaser)((*WindowsNode)(nil))
+
+// Release decreases the open count. The kernel doesn't wait with
+// returning from close(), so if the caller is too quick to
+// unlink/rename after calling close(), this may still trigger EBUSY.
+func (n *WindowsNode) Release(ctx context.Context, f fs.FileHandle) syscall.Errno {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+
+	n.openCount--
+	if fr, ok := f.(fs.FileReleaser); ok {
+		return fr.Release(ctx)
+	}
+	return 0
+}
+
+func isBusy(parent *fs.Inode, name string) bool {
+	if ch := parent.GetChild(name); ch != nil {
+		if wn, ok := ch.Operations().(*WindowsNode); ok {
+			wn.mu.Lock()
+			defer wn.mu.Unlock()
+			if wn.openCount > 0 {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+var _ = (fs.NodeUnlinker)((*WindowsNode)(nil))
+
+func (n *WindowsNode) Unlink(ctx context.Context, name string) syscall.Errno {
+	if isBusy(n.EmbeddedInode(), name) {
+		return syscall.EBUSY
+	}
+
+	return n.LoopbackNode.Unlink(ctx, name)
+}
+
+func newWindowsNode(rootData *fs.LoopbackRoot, parent *fs.Inode, name string, st *syscall.Stat_t) fs.InodeEmbedder {
+	n := &WindowsNode{
+		LoopbackNode: fs.LoopbackNode{
+			RootData: rootData,
+		},
+	}
+	return n
+}
+
+// ExampleLoopbackReuse shows how to build a file system on top of the
+// loopback file system.
+func Example_loopbackReuse() {
+	mntDir := "/tmp/mnt"
+	origDir := "/tmp/orig"
+
+	rootData := &fs.LoopbackRoot{
+		NewNode: newWindowsNode,
+		Path:    origDir,
+	}
+
+	sec := time.Second
+	opts := &fs.Options{
+		AttrTimeout:  &sec,
+		EntryTimeout: &sec,
+	}
+
+	server, err := fs.Mount(mntDir, newWindowsNode(rootData, nil, "", nil), opts)
+	if err != nil {
+		log.Fatalf("Mount fail: %v\n", err)
+	}
+	fmt.Printf("files under %s cannot be deleted if they are opened", mntDir)
+	server.Wait()
+}

fs/windows_test.go

+// Copyright 2020 the Go-FUSE Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fs_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/hanwen/go-fuse/v2/fs"
+	"github.com/hanwen/go-fuse/v2/internal/testutil"
+)
+
+func TestWindowsEmulations(t *testing.T) {
+	mntDir, err := ioutil.TempDir("", "ZipFS")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(mntDir)
+	origDir, err := ioutil.TempDir("", "ZipFS")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(origDir)
+
+	rootData := &fs.LoopbackRoot{
+		NewNode: newWindowsNode,
+		Path:    origDir,
+	}
+	opts := fs.Options{}
+	opts.Debug = testutil.VerboseTest()
+	server, err := fs.Mount(mntDir, newWindowsNode(rootData, nil, "", nil), &opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer server.Unmount()
+
+	data := []byte("hello")
+	nm := mntDir + "/file"
+	if err := ioutil.WriteFile(nm, data, 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	if got, err := ioutil.ReadFile(nm); err != nil {
+		t.Fatal(err)
+	} else if bytes.Compare(got, data) != 0 {
+		t.Fatalf("got %q want %q", got, data)
+	}
+
+	f, err := os.Open(nm)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := syscall.Unlink(nm); err == nil {
+		t.Fatal("Unlink should have failed")
+	}
+
+	f.Close()
+	// Ugh - it may take a while for the RELEASE to be processed.
+	time.Sleep(10 * time.Millisecond)
+
+	if err := syscall.Unlink(nm); err != nil {
+		t.Fatalf("Unlink: %v", err)
+	}
+}

fuse/api.go

@@ -67,6 +67,12 @@
 // Typically, each call of the API happens in its own
 // goroutine, so take care to make the file system thread-safe.
 //
+// Be careful when you access the FUSE mount from the same process. An access can
+// tie up two OS threads (one on the request side and one on the FUSE server side).
+// This can deadlock if there is no free thread to handle the FUSE server side.
+// Run your program with GOMAXPROCS=1 to make the problem easier to reproduce,
+// see https://github.com/hanwen/go-fuse/issues/261 for an example of that
+// problem.
 //
 // Higher level interfaces
 //

fuse/mount_darwin.go

@@ -9,89 +9,125 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"strings"
 	"syscall"
+	"unsafe"
 )
 
-func openFUSEDevice() (*os.File, error) {
-	fs, err := filepath.Glob("/dev/osxfuse*")
+func unixgramSocketpair() (l, r *os.File, err error) {
+	fd, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
 	if err != nil {
-		return nil, err
+		return nil, nil, os.NewSyscallError("socketpair",
+			err.(syscall.Errno))
 	}
-	if len(fs) == 0 {
-		bin := oldLoadBin
-		if _, err := os.Stat(newLoadBin); err == nil {
-			bin = newLoadBin
-		}
-
-		cmd := exec.Command(bin)
-		if err := cmd.Run(); err != nil {
-			return nil, err
-		}
-		fs, err = filepath.Glob("/dev/osxfuse*")
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	for _, fn := range fs {
-		f, err := os.OpenFile(fn, os.O_RDWR, 0)
-		if err != nil {
-			continue
-		}
-		return f, nil
-	}
-
-	return nil, fmt.Errorf("all FUSE devices busy")
+	l = os.NewFile(uintptr(fd[0]), "socketpair-half1")
+	r = os.NewFile(uintptr(fd[1]), "socketpair-half2")
+	return
 }
 
-const oldLoadBin = "/Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs"
-const newLoadBin = "/Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse"
-
-const oldMountBin = "/Library/Filesystems/osxfusefs.fs/Support/mount_osxfusefs"
-const newMountBin = "/Library/Filesystems/osxfuse.fs/Contents/Resources/mount_osxfuse"
-
+// Create a FUSE FS on the specified mount point.  The returned
+// mount point is always absolute.
 func mount(mountPoint string, opts *MountOptions, ready chan<- error) (fd int, err error) {
-	f, err := openFUSEDevice()
+	local, remote, err := unixgramSocketpair()
 	if err != nil {
-		return 0, err
+		return
 	}
 
-	bin := oldMountBin
-	if _, err := os.Stat(newMountBin); err == nil {
-		bin = newMountBin
+	defer local.Close()
+	defer remote.Close()
+
+	bin, err := fusermountBinary()
+	if err != nil {
+		return 0, err
 	}
 
-	cmd := exec.Command(bin, "-o", strings.Join(opts.optionsStrings(), ","), "-o", fmt.Sprintf("iosize=%d", opts.MaxWrite), "3", mountPoint)
-	cmd.ExtraFiles = []*os.File{f}
-	cmd.Env = append(os.Environ(), "MOUNT_FUSEFS_CALL_BY_LIB=", "MOUNT_OSXFUSE_CALL_BY_LIB=",
-		"MOUNT_OSXFUSE_DAEMON_PATH="+os.Args[0],
-		"MOUNT_FUSEFS_DAEMON_PATH="+os.Args[0])
+	cmd := exec.Command(bin,
+		"-o", strings.Join(opts.optionsStrings(), ","),
+		"-o", fmt.Sprintf("iosize=%d", opts.MaxWrite),
+		mountPoint)
+	cmd.ExtraFiles = []*os.File{remote} // fd would be (index + 3)
+	cmd.Env = append(os.Environ(),
+		"_FUSE_CALL_BY_LIB=",
+		"_FUSE_DAEMON_PATH="+os.Args[0],
+		"_FUSE_COMMFD=3",
+		"_FUSE_COMMVERS=2",
+		"MOUNT_OSXFUSE_CALL_BY_LIB=",
+		"MOUNT_OSXFUSE_DAEMON_PATH="+os.Args[0])
 
 	var out, errOut bytes.Buffer
 	cmd.Stdout = &out
 	cmd.Stderr = &errOut
 
-	if err := cmd.Start(); err != nil {
-		f.Close()
-		return 0, err
+	if err = cmd.Start(); err != nil {
+		return
 	}
+
+	fd, err = getConnection(local)
+	if err != nil {
+		return -1, err
+	}
+
 	go func() {
-		err := cmd.Wait()
-		if err != nil {
-			err = fmt.Errorf("mount_osxfusefs failed: %v. Stderr: %s, Stdout: %s", err, errOut.String(), out.String())
+		// wait inside a goroutine or otherwise it would block forever for unknown reasons
+		if err := cmd.Wait(); err != nil {
+			err = fmt.Errorf("mount_osxfusefs failed: %v. Stderr: %s, Stdout: %s",
+				err, errOut.String(), out.String())
 		}
 
 		ready <- err
 		close(ready)
 	}()
 
-	// The finalizer for f will close its fd so we return a dup.
-	defer f.Close()
-	return syscall.Dup(int(f.Fd()))
+	// golang sets CLOEXEC on file descriptors when they are
+	// acquired through normal operations (e.g. open).
+	// Buf for fd, we have to set CLOEXEC manually
+	syscall.CloseOnExec(fd)
+
+	return fd, err
 }
 
 func unmount(dir string, opts *MountOptions) error {
 	return syscall.Unmount(dir, 0)
 }
+
+func getConnection(local *os.File) (int, error) {
+	var data [4]byte
+	control := make([]byte, 4*256)
+
+	// n, oobn, recvflags, from, errno  - todo: error checking.
+	_, oobn, _, _,
+		err := syscall.Recvmsg(
+		int(local.Fd()), data[:], control[:], 0)
+	if err != nil {
+		return 0, err
+	}
+
+	message := *(*syscall.Cmsghdr)(unsafe.Pointer(&control[0]))
+	fd := *(*int32)(unsafe.Pointer(uintptr(unsafe.Pointer(&control[0])) + syscall.SizeofCmsghdr))
+
+	if message.Type != syscall.SCM_RIGHTS {
+		return 0, fmt.Errorf("getConnection: recvmsg returned wrong control type: %d", message.Type)
+	}
+	if oobn <= syscall.SizeofCmsghdr {
+		return 0, fmt.Errorf("getConnection: too short control message. Length: %d", oobn)
+	}
+	if fd < 0 {
+		return 0, fmt.Errorf("getConnection: fd < 0: %d", fd)
+	}
+	return int(fd), nil
+}
+
+func fusermountBinary() (string, error) {
+	binPaths := []string{
+		"/Library/Filesystems/macfuse.fs/Contents/Resources/mount_macfuse",
+		"/Library/Filesystems/osxfuse.fs/Contents/Resources/mount_osxfuse",
+	}
+
+	for _, path := range binPaths {
+		if _, err := os.Stat(path); err == nil {
+			return path, nil
+		}
+	}
+
+	return "", fmt.Errorf("no FUSE mount utility found")
+}

fuse/opcode.go

@@ -314,7 +314,7 @@ func doBatchForget(server *Server, req *request) {
 	forgets := *(*[]_ForgetOne)(unsafe.Pointer(h))
 	for i, f := range forgets {
 		if server.opts.Debug {
-			log.Printf("doBatchForget: rx %d %d/%d: FORGET i%d {Nlookup=%d}",
+			log.Printf("doBatchForget: rx %d %d/%d: FORGET n%d {Nlookup=%d}",
 				req.inHeader.Unique, i+1, len(forgets), f.NodeId, f.Nlookup)
 		}
 		if f.NodeId == pollHackInode {

fuse/poll.go

@@ -28,15 +28,22 @@ func doPollHackLookup(ms *Server, req *request) {
 			Fh: pollHackInode,
 		}
 		req.status = OK
-	case _OP_GETATTR:
+	case _OP_GETATTR, _OP_SETATTR:
 		out := (*AttrOut)(req.outData())
 		out.Attr = attr
 		req.status = OK
 	case _OP_POLL:
 		req.status = ENOSYS
+
+	case _OP_ACCESS, _OP_FLUSH, _OP_RELEASE:
+		// Avoid upsetting the OSX mount process.
+		req.status = OK
 	default:
 		// We want to avoid switching off features through our
-		// poll hack, so don't use ENOSYS
+		// poll hack, so don't use ENOSYS. It would be nice if
+		// we could transmit no error code at all, but for
+		// some opcodes, we'd have to invent credible data to
+		// return as well.
 		req.status = ERANGE
 	}
 }

fuse/print.go

@@ -214,13 +214,13 @@ func ft(tsec uint64, tnsec uint32) float64 {
 
 // Returned by LOOKUP
 func (o *EntryOut) string() string {
-	return fmt.Sprintf("{i%d g%d tE=%gs tA=%gs %v}",
+	return fmt.Sprintf("{n%d g%d tE=%gs tA=%gs %v}",
 		o.NodeId, o.Generation, ft(o.EntryValid, o.EntryValidNsec),
 		ft(o.AttrValid, o.AttrValidNsec), &o.Attr)
 }
 
 func (o *CreateOut) string() string {
-	return fmt.Sprintf("{i%d g%d %v %v}", o.NodeId, o.Generation, &o.EntryOut, &o.OpenOut)
+	return fmt.Sprintf("{n%d g%d %v %v}", o.NodeId, o.Generation, &o.EntryOut, &o.OpenOut)
 }
 
 func (o *StatfsOut) string() string {
@@ -243,11 +243,11 @@ func (o *NotifyInvalDeleteOut) string() string {
 }
 
 func (o *NotifyStoreOut) string() string {
-	return fmt.Sprintf("{i%d [%d +%d)}", o.Nodeid, o.Offset, o.Size)
+	return fmt.Sprintf("{n%d [%d +%d)}", o.Nodeid, o.Offset, o.Size)
 }
 
 func (o *NotifyRetrieveOut) string() string {
-	return fmt.Sprintf("{> %d: i%d [%d +%d)}", o.NotifyUnique, o.Nodeid, o.Offset, o.Size)
+	return fmt.Sprintf("{> %d: n%d [%d +%d)}", o.NotifyUnique, o.Nodeid, o.Offset, o.Size)
 }
 
 func (i *NotifyRetrieveIn) string() string {
@@ -260,7 +260,7 @@ func (f *FallocateIn) string() string {
 }
 
 func (f *LinkIn) string() string {
-	return fmt.Sprintf("{Oldnodeid: %d}", f.Oldnodeid)
+	return fmt.Sprintf("{Oldnodeid: n%d}", f.Oldnodeid)
 }
 
 func (o *WriteOut) string() string {
@@ -268,7 +268,7 @@ func (o *WriteOut) string() string {
 
 }
 func (i *CopyFileRangeIn) string() string {
-	return fmt.Sprintf("{Fh %d [%d +%d) => i%d Fh %d [%d, %d)}",
+	return fmt.Sprintf("{Fh %d [%d +%d) => n%d Fh %d [%d, %d)}",
 		i.FhIn, i.OffIn, i.Len, i.NodeIdOut, i.FhOut, i.OffOut, i.Len)
 }
 

fuse/request.go

@@ -103,7 +103,7 @@ func (r *request) InputDebug() string {
 		names += fmt.Sprintf("%s %db", data, len(r.arg))
 	}
 
-	return fmt.Sprintf("rx %d: %s i%d %s%s",
+	return fmt.Sprintf("rx %d: %s n%d %s%s",
 		r.inHeader.Unique, operationName(r.inHeader.Opcode), r.inHeader.NodeId,
 		val, names)
 }

fuse/server.go

@@ -21,6 +21,9 @@ import (
 const (
 	// The kernel caps writes at 128k.
 	MAX_KERNEL_WRITE = 128 * 1024
+
+	minMaxReaders = 2
+	maxMaxReaders = 16
 )
 
 // Server contains the logic for reading from the FUSE device and
@@ -40,6 +43,9 @@ type Server struct {
 
 	opts *MountOptions
 
+	// maxReaders is the maximum number of goroutines reading requests
+	maxReaders int
+
 	// Pools for []byte
 	buffers bufferPool
 
@@ -161,9 +167,17 @@ func NewServer(fs RawFileSystem, mountPoint string, opts *MountOptions) (*Server
 		}
 	}
 
+	maxReaders := runtime.GOMAXPROCS(0)
+	if maxReaders < minMaxReaders {
+		maxReaders = minMaxReaders
+	} else if maxReaders > maxMaxReaders {
+		maxReaders = maxMaxReaders
+	}
+
 	ms := &Server{
 		fileSystem:  fs,
 		opts:        &o,
+		maxReaders:  maxReaders,
 		retrieveTab: make(map[uint64]*retrieveCacheRequest),
 		// OSX has races when multiple routines read from the
 		// FUSE device: on unmount, sometime some reads do not
@@ -238,9 +252,6 @@ func (ms *Server) DebugData() string {
 	return fmt.Sprintf("readers: %d", r)
 }
 
-// What is a good number?  Maybe the number of CPUs?
-const _MAX_READERS = 2
-
 // handleEINTR retries the given function until it doesn't return syscall.EINTR.
 // This is similar to the HANDLE_EINTR() macro from Chromium ( see
 // https://code.google.com/p/chromium/codesearch#chromium/src/base/posix/eintr_wrapper.h
@@ -267,7 +278,7 @@ func (ms *Server) readRequest(exitIdle bool) (req *request, code Status) {
 	dest := ms.readPool.Get().([]byte)
 
 	ms.reqMu.Lock()
-	if ms.reqReaders > _MAX_READERS {
+	if ms.reqReaders > ms.maxReaders {
 		ms.reqMu.Unlock()
 		return nil, OK
 	}

fuse/test/cachecontrol_test.go

@@ -189,11 +189,19 @@ func TestCacheControl(t *testing.T) {
 	defer func() {
 		xmunmap(fmmap)
 	}()
-	xmlock(fmmap)
 
 	// assertMmapRead asserts that file's mmaped memory reads as dataOK.
 	assertMmapRead := func(subj, dataOK string) {
 		t.Helper()
+		// Use the Mlock() syscall to get the mmap'ed range into the kernel
+		// cache again, triggering FUSE reads as neccessary. A blocked syscall does
+		// not count towards GOMAXPROCS, so there should be a thread available
+		// to handle the FUSE reads.
+		// If we don't Mlock() first, the memory comparison triggers a page
+		// fault, which blocks the thread, and deadlocks the test reliably at
+		// GOMAXPROCS=1.
+		// Fixes https://github.com/hanwen/go-fuse/issues/261 .
+		xmlock(fmmap)
 		if string(fmmap) != dataOK {
 			t.Fatalf("%s: file mmap: got %q  ; want %q", subj, fmmap, dataOK)
 		}

go.mod

@@ -3,6 +3,7 @@ module github.com/hanwen/go-fuse/v2
 require (
 	github.com/hanwen/go-fuse v1.0.0
 	github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348
+	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
 	golang.org/x/sys v0.0.0-20180830151530-49385e6e1522
 )
 

go.sum

@@ -2,5 +2,7 @@ github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc=
 github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 h1:Ve1ORMCxvRmSXBwJK+t3Oy+V2vRW2OetUQBq4rJIkZE=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=