buffer: refactor to a form that won't require memclr.

Necessary because our evil memclr should go away.

Benchmarks:

    benchmark                                          old ns/op     new ns/op     delta
    BenchmarkOutMessageReset/Single_buffer-12          4.94          4.95          +0.20%
    BenchmarkOutMessageReset/Many_buffers-12           10.4          12.0          +15.38%
    BenchmarkOutMessageGrowShrink/Single_buffer-12     42861         47666         +11.21%
    BenchmarkOutMessageGrowShrink/Many_buffers-12      101896        100626        -1.25%

    benchmark                                          old MB/s     new MB/s     speedup
    BenchmarkOutMessageReset/Single_buffer-12          4853.85      4851.64      1.00x
    BenchmarkOutMessageReset/Many_buffers-12           2311.82      2006.72      0.87x
    BenchmarkOutMessageGrowShrink/Single_buffer-12     24464.39     21998.19     0.90x
    BenchmarkOutMessageGrowShrink/Many_buffers-12      10290.60     10420.48     1.01x

conversions.go

@@ -279,7 +279,7 @@ func convertInMessage(
 		o = to
 
 		readSize := int(in.Size)
-		p := outMsg.GrowNoZero(uintptr(readSize))
+		p := outMsg.GrowNoZero(readSize)
 		if p == nil {
 			err = fmt.Errorf("Can't grow for %d-byte read", readSize)
 			return
@@ -305,7 +305,7 @@ func convertInMessage(
 		o = to
 
 		readSize := int(in.Size)
-		p := outMsg.GrowNoZero(uintptr(readSize))
+		p := outMsg.GrowNoZero(readSize)
 		if p == nil {
 			err = fmt.Errorf("Can't grow for %d-byte read", readSize)
 			return
@@ -469,7 +469,7 @@ func (c *Connection) kernelResponse(
 		// the header, because on OS X the kernel otherwise returns EINVAL when we
 		// attempt to write an error response with a length that extends beyond the
 		// header.
-		m.ShrinkTo(buffer.OutMessageInitialSize)
+		m.ShrinkTo(buffer.OutMessageHeaderSize)
 	}
 
 	// Otherwise, fill in the rest of the response.
@@ -489,45 +489,45 @@ func (c *Connection) kernelResponseForOp(
 	// Create the appropriate output message
 	switch o := op.(type) {
 	case *fuseops.LookUpInodeOp:
-		size := fusekernel.EntryOutSize(c.protocol)
+		size := int(fusekernel.EntryOutSize(c.protocol))
 		out := (*fusekernel.EntryOut)(m.Grow(size))
 		convertChildInodeEntry(&o.Entry, out)
 
 	case *fuseops.GetInodeAttributesOp:
-		size := fusekernel.AttrOutSize(c.protocol)
+		size := int(fusekernel.AttrOutSize(c.protocol))
 		out := (*fusekernel.AttrOut)(m.Grow(size))
 		out.AttrValid, out.AttrValidNsec = convertExpirationTime(
 			o.AttributesExpiration)
 		convertAttributes(o.Inode, &o.Attributes, &out.Attr)
 
 	case *fuseops.SetInodeAttributesOp:
-		size := fusekernel.AttrOutSize(c.protocol)
+		size := int(fusekernel.AttrOutSize(c.protocol))
 		out := (*fusekernel.AttrOut)(m.Grow(size))
 		out.AttrValid, out.AttrValidNsec = convertExpirationTime(
 			o.AttributesExpiration)
 		convertAttributes(o.Inode, &o.Attributes, &out.Attr)
 
 	case *fuseops.MkDirOp:
-		size := fusekernel.EntryOutSize(c.protocol)
+		size := int(fusekernel.EntryOutSize(c.protocol))
 		out := (*fusekernel.EntryOut)(m.Grow(size))
 		convertChildInodeEntry(&o.Entry, out)
 
 	case *fuseops.MkNodeOp:
-		size := fusekernel.EntryOutSize(c.protocol)
+		size := int(fusekernel.EntryOutSize(c.protocol))
 		out := (*fusekernel.EntryOut)(m.Grow(size))
 		convertChildInodeEntry(&o.Entry, out)
 
 	case *fuseops.CreateFileOp:
-		eSize := fusekernel.EntryOutSize(c.protocol)
+		eSize := int(fusekernel.EntryOutSize(c.protocol))
 
 		e := (*fusekernel.EntryOut)(m.Grow(eSize))
 		convertChildInodeEntry(&o.Entry, e)
 
-		oo := (*fusekernel.OpenOut)(m.Grow(unsafe.Sizeof(fusekernel.OpenOut{})))
+		oo := (*fusekernel.OpenOut)(m.Grow(int(unsafe.Sizeof(fusekernel.OpenOut{}))))
 		oo.Fh = uint64(o.Handle)
 
 	case *fuseops.CreateSymlinkOp:
-		size := fusekernel.EntryOutSize(c.protocol)
+		size := int(fusekernel.EntryOutSize(c.protocol))
 		out := (*fusekernel.EntryOut)(m.Grow(size))
 		convertChildInodeEntry(&o.Entry, out)
 
@@ -541,20 +541,20 @@ func (c *Connection) kernelResponseForOp(
 		// Empty response
 
 	case *fuseops.OpenDirOp:
-		out := (*fusekernel.OpenOut)(m.Grow(unsafe.Sizeof(fusekernel.OpenOut{})))
+		out := (*fusekernel.OpenOut)(m.Grow(int(unsafe.Sizeof(fusekernel.OpenOut{}))))
 		out.Fh = uint64(o.Handle)
 
 	case *fuseops.ReadDirOp:
 		// convertInMessage already set up the destination buffer to be at the end
 		// of the out message. We need only shrink to the right size based on how
 		// much the user read.
-		m.ShrinkTo(buffer.OutMessageInitialSize + uintptr(o.BytesRead))
+		m.ShrinkTo(buffer.OutMessageHeaderSize + o.BytesRead)
 
 	case *fuseops.ReleaseDirHandleOp:
 		// Empty response
 
 	case *fuseops.OpenFileOp:
-		out := (*fusekernel.OpenOut)(m.Grow(unsafe.Sizeof(fusekernel.OpenOut{})))
+		out := (*fusekernel.OpenOut)(m.Grow(int(unsafe.Sizeof(fusekernel.OpenOut{}))))
 		out.Fh = uint64(o.Handle)
 
 		if o.KeepPageCache {
@@ -565,10 +565,10 @@ func (c *Connection) kernelResponseForOp(
 		// convertInMessage already set up the destination buffer to be at the end
 		// of the out message. We need only shrink to the right size based on how
 		// much the user read.
-		m.ShrinkTo(buffer.OutMessageInitialSize + uintptr(o.BytesRead))
+		m.ShrinkTo(buffer.OutMessageHeaderSize + o.BytesRead)
 
 	case *fuseops.WriteFileOp:
-		out := (*fusekernel.WriteOut)(m.Grow(unsafe.Sizeof(fusekernel.WriteOut{})))
+		out := (*fusekernel.WriteOut)(m.Grow(int(unsafe.Sizeof(fusekernel.WriteOut{}))))
 		out.Size = uint32(len(o.Data))
 
 	case *fuseops.SyncFileOp:
@@ -584,7 +584,7 @@ func (c *Connection) kernelResponseForOp(
 		m.AppendString(o.Target)
 
 	case *fuseops.StatFSOp:
-		out := (*fusekernel.StatfsOut)(m.Grow(unsafe.Sizeof(fusekernel.StatfsOut{})))
+		out := (*fusekernel.StatfsOut)(m.Grow(int(unsafe.Sizeof(fusekernel.StatfsOut{}))))
 		out.St.Blocks = o.Blocks
 		out.St.Bfree = o.BlocksFree
 		out.St.Bavail = o.BlocksAvailable
@@ -620,7 +620,7 @@ func (c *Connection) kernelResponseForOp(
 		out.St.Frsize = o.BlockSize
 
 	case *initOp:
-		out := (*fusekernel.InitOut)(m.Grow(unsafe.Sizeof(fusekernel.InitOut{})))
+		out := (*fusekernel.InitOut)(m.Grow(int(unsafe.Sizeof(fusekernel.InitOut{}))))
 
 		out.Major = o.Library.Major
 		out.Minor = o.Library.Minor

internal/buffer/out_message.go

@@ -23,14 +23,9 @@ import (
 	"github.com/jacobsa/fuse/internal/fusekernel"
 )
 
-const outHeaderSize = unsafe.Sizeof(fusekernel.OutHeader{})
-
-// OutMessage structs begin life with Len() == OutMessageInitialSize.
-const OutMessageInitialSize = outHeaderSize
-
-// We size out messages to be large enough to hold a header for the response
-// plus the largest read that may come in.
-const outMessageSize = outHeaderSize + MaxReadSize
+// OutMessageHeaderSize is the size of the leading header in every
+// properly-constructed OutMessage. Reset brings the message back to this size.
+const OutMessageHeaderSize = int(unsafe.Sizeof(fusekernel.OutHeader{}))
 
 // OutMessage provides a mechanism for constructing a single contiguous fuse
 // message from multiple segments, where the first segment is always a
@@ -38,72 +33,81 @@ const outMessageSize = outHeaderSize + MaxReadSize
 //
 // Must be initialized with Reset.
 type OutMessage struct {
-	offset  uintptr
-	storage [outMessageSize]byte
+	// The offset into payload to which we're currently writing.
+	payloadOffset int
+
+	header  fusekernel.OutHeader
+	payload [MaxReadSize]byte
 }
 
-// Make sure alignment works out correctly, at least for the header.
+// Make sure that the header and payload are contiguous.
 func init() {
-	a := unsafe.Alignof(OutMessage{})
-	o := unsafe.Offsetof(OutMessage{}.storage)
-	e := unsafe.Alignof(fusekernel.OutHeader{})
+	a := unsafe.Offsetof(OutMessage{}.header) + uintptr(OutMessageHeaderSize)
+	b := unsafe.Offsetof(OutMessage{}.payload)
 
-	if a%e != 0 || o%e != 0 {
-		log.Panicf("Bad alignment or offset: %d, %d, need %d", a, o, e)
+	if a != b {
+		log.Panicf(
+			"header ends at offset %d, but payload starts at offset %d",
+			a, b)
 	}
 }
 
-// Reset the message so that it is ready to be used again. Afterward, the
-// contents are solely a zeroed header.
+// Reset resets m so that it's ready to be used again. Afterward, the contents
+// are solely a zeroed fusekernel.OutHeader struct.
 func (m *OutMessage) Reset() {
-	m.offset = OutMessageInitialSize
-	memclr(unsafe.Pointer(&m.storage), OutMessageInitialSize)
+	m.payloadOffset = 0
+	memclr(unsafe.Pointer(&m.header), uintptr(OutMessageHeaderSize))
 }
 
-// Return a pointer to the header at the start of the message.
-func (b *OutMessage) OutHeader() (h *fusekernel.OutHeader) {
-	h = (*fusekernel.OutHeader)(unsafe.Pointer(&b.storage))
-	return
+// OutHeader returns a pointer to the header at the start of the message.
+func (m *OutMessage) OutHeader() *fusekernel.OutHeader {
+	return &m.header
 }
 
-// Grow the buffer by the supplied number of bytes, returning a pointer to the
-// start of the new segment, which is zeroed. If there is no space left, return
-// the nil pointer.
-func (b *OutMessage) Grow(size uintptr) (p unsafe.Pointer) {
-	p = b.GrowNoZero(size)
+// Grow grows m's buffer by the given number of bytes, returning a pointer to
+// the start of the new segment, which is guaranteed to be zeroed. If there is
+// insufficient space, it returns nil.
+func (m *OutMessage) Grow(n int) (p unsafe.Pointer) {
+	p = m.GrowNoZero(n)
 	if p != nil {
-		memclr(p, size)
+		memclr(p, uintptr(n))
 	}
 
 	return
 }
 
-// Equivalent to Grow, except the new segment is not zeroed. Use with caution!
-func (b *OutMessage) GrowNoZero(size uintptr) (p unsafe.Pointer) {
-	if outMessageSize-b.offset < size {
+// GrowNoZero is equivalent to Grow, except the new segment is not zeroed. Use
+// with caution!
+func (m *OutMessage) GrowNoZero(n int) (p unsafe.Pointer) {
+	// Will we overflow the buffer?
+	o := m.payloadOffset
+	if len(m.payload)-o < n {
 		return
 	}
 
-	p = unsafe.Pointer(uintptr(unsafe.Pointer(&b.storage)) + b.offset)
-	b.offset += size
+	p = unsafe.Pointer(uintptr(unsafe.Pointer(&m.payload)) + uintptr(o))
+	m.payloadOffset = o + n
 
 	return
 }
 
-// Shrink to the supplied size. Panic if the size is greater than Len() or less
-// than OutMessageInitialSize.
-func (b *OutMessage) ShrinkTo(n uintptr) {
-	if n < OutMessageInitialSize || n > b.offset {
-		panic(fmt.Sprintf("ShrinkTo(%d) out of range for offset %d", n, b.offset))
+// ShrinkTo shrinks m to the given size. It panics if the size is greater than
+// Len() or less than OutMessageHeaderSize.
+func (m *OutMessage) ShrinkTo(n int) {
+	if n < OutMessageHeaderSize || n > m.Len() {
+		panic(fmt.Sprintf(
+			"ShrinkTo(%d) out of range (current Len: %d)",
+			n,
+			m.Len()))
 	}
 
-	b.offset = n
+	m.payloadOffset = n - OutMessageHeaderSize
 }
 
-// Equivalent to growing by the length of p, then copying p over the new
-// segment. Panics if there is not enough room available.
-func (b *OutMessage) Append(src []byte) {
-	p := b.GrowNoZero(uintptr(len(src)))
+// Append is equivalent to growing by len(src), then copying src over the new
+// segment. Int panics if there is not enough room available.
+func (m *OutMessage) Append(src []byte) {
+	p := m.GrowNoZero(len(src))
 	if p == nil {
 		panic(fmt.Sprintf("Can't grow %d bytes", len(src)))
 	}
@@ -114,10 +118,9 @@ func (b *OutMessage) Append(src []byte) {
 	return
 }
 
-// Equivalent to growing by the length of s, then copying s over the new
-// segment. Panics if there is not enough room available.
-func (b *OutMessage) AppendString(src string) {
-	p := b.GrowNoZero(uintptr(len(src)))
+// AppendString is like Append, but accepts string input.
+func (m *OutMessage) AppendString(src string) {
+	p := m.GrowNoZero(len(src))
 	if p == nil {
 		panic(fmt.Sprintf("Can't grow %d bytes", len(src)))
 	}
@@ -128,12 +131,20 @@ func (b *OutMessage) AppendString(src string) {
 	return
 }
 
-// Return the current size of the buffer.
-func (b *OutMessage) Len() int {
-	return int(b.offset)
+// Len returns the current size of the message, including the leading header.
+func (m *OutMessage) Len() int {
+	return OutMessageHeaderSize + m.payloadOffset
 }
 
-// Return a reference to the current contents of the buffer.
-func (b *OutMessage) Bytes() []byte {
-	return b.storage[:int(b.offset)]
+// Bytes returns a reference to the current contents of the buffer, including
+// the leading header.
+func (m *OutMessage) Bytes() []byte {
+	l := m.Len()
+	sh := reflect.SliceHeader{
+		Data: uintptr(unsafe.Pointer(&m.header)),
+		Len:  l,
+		Cap:  l,
+	}
+
+	return *(*[]byte)(unsafe.Pointer(&sh))
 }

internal/buffer/out_message_test.go

@@ -101,7 +101,7 @@ func TestOutMessageAppend(t *testing.T) {
 	om.Append(wantPayload[4:])
 
 	// The result should be a zeroed header followed by the desired payload.
-	const wantLen = int(OutMessageInitialSize) + len(wantPayloadStr)
+	const wantLen = OutMessageHeaderSize + len(wantPayloadStr)
 
 	if got, want := om.Len(), wantLen; got != want {
 		t.Errorf("om.Len() = %d, want %d", got, want)
@@ -113,7 +113,7 @@ func TestOutMessageAppend(t *testing.T) {
 	}
 
 	want := append(
-		make([]byte, OutMessageInitialSize),
+		make([]byte, OutMessageHeaderSize),
 		wantPayload...)
 
 	if !bytes.Equal(b, want) {
@@ -131,7 +131,7 @@ func TestOutMessageAppendString(t *testing.T) {
 	om.AppendString(wantPayload[4:])
 
 	// The result should be a zeroed header followed by the desired payload.
-	const wantLen = int(OutMessageInitialSize) + len(wantPayload)
+	const wantLen = OutMessageHeaderSize + len(wantPayload)
 
 	if got, want := om.Len(), wantLen; got != want {
 		t.Errorf("om.Len() = %d, want %d", got, want)
@@ -143,7 +143,7 @@ func TestOutMessageAppendString(t *testing.T) {
 	}
 
 	want := append(
-		make([]byte, OutMessageInitialSize),
+		make([]byte, OutMessageHeaderSize),
 		wantPayload...)
 
 	if !bytes.Equal(b, want) {
@@ -159,10 +159,10 @@ func TestOutMessageShrinkTo(t *testing.T) {
 	om.AppendString("burrito")
 
 	// Shrink it.
-	om.ShrinkTo(OutMessageInitialSize + uintptr(len("taco")))
+	om.ShrinkTo(OutMessageHeaderSize + len("taco"))
 
 	// The result should be a zeroed header followed by "taco".
-	const wantLen = int(OutMessageInitialSize) + len("taco")
+	const wantLen = OutMessageHeaderSize + len("taco")
 
 	if got, want := om.Len(), wantLen; got != want {
 		t.Errorf("om.Len() = %d, want %d", got, want)
@@ -174,7 +174,7 @@ func TestOutMessageShrinkTo(t *testing.T) {
 	}
 
 	want := append(
-		make([]byte, OutMessageInitialSize),
+		make([]byte, OutMessageHeaderSize),
 		"taco"...)
 
 	if !bytes.Equal(b, want) {
@@ -233,7 +233,7 @@ func TestOutMessageReset(t *testing.T) {
 		om.Reset()
 
 		// Check that the length was updated.
-		if got, want := int(om.Len()), int(OutMessageInitialSize); got != want {
+		if got, want := om.Len(), OutMessageHeaderSize; got != want {
 			t.Fatalf("om.Len() = %d, want %d", got, want)
 		}
 
@@ -269,7 +269,7 @@ func TestOutMessageGrow(t *testing.T) {
 			t.Fatalf("fillWithGarbage: %v", err)
 		}
 
-		om.ShrinkTo(OutMessageInitialSize)
+		om.ShrinkTo(OutMessageHeaderSize)
 	}
 
 	// Call Grow.
@@ -278,7 +278,7 @@ func TestOutMessageGrow(t *testing.T) {
 	}
 
 	// Check the resulting length in two ways.
-	const wantLen = int(payloadSize + OutMessageInitialSize)
+	const wantLen = payloadSize + OutMessageHeaderSize
 	if got, want := om.Len(), wantLen; got != want {
 		t.Errorf("om.Len() = %d, want %d", got)
 	}
@@ -289,7 +289,7 @@ func TestOutMessageGrow(t *testing.T) {
 	}
 
 	// Check that the payload was zeroed.
-	for i, x := range b[OutMessageInitialSize:] {
+	for i, x := range b[OutMessageHeaderSize:] {
 		if x != 0 {
 			t.Fatalf("non-zero byte 0x%02x at payload offset %d", x, i)
 		}
@@ -304,7 +304,7 @@ func BenchmarkOutMessageReset(b *testing.B) {
 			om.Reset()
 		}
 
-		b.SetBytes(int64(unsafe.Offsetof(om.storage)) + int64(om.offset))
+		b.SetBytes(int64(unsafe.Offsetof(om.payload)))
 	})
 
 	// Many megabytes worth of buffers, which should defeat the CPU cache.
@@ -321,7 +321,7 @@ func BenchmarkOutMessageReset(b *testing.B) {
 			oms[i%numMessages].Reset()
 		}
 
-		b.SetBytes(int64(unsafe.Offsetof(oms[0].storage)) + int64(oms[0].offset))
+		b.SetBytes(int64(unsafe.Offsetof(oms[0].payload)))
 	})
 }
 
@@ -331,7 +331,7 @@ func BenchmarkOutMessageGrowShrink(b *testing.B) {
 		var om OutMessage
 		for i := 0; i < b.N; i++ {
 			om.Grow(MaxReadSize)
-			om.ShrinkTo(OutMessageInitialSize)
+			om.ShrinkTo(OutMessageHeaderSize)
 		}
 
 		b.SetBytes(int64(MaxReadSize))
@@ -349,7 +349,7 @@ func BenchmarkOutMessageGrowShrink(b *testing.B) {
 
 		for i := 0; i < b.N; i++ {
 			oms[i%numMessages].Grow(MaxReadSize)
-			oms[i%numMessages].ShrinkTo(OutMessageInitialSize)
+			oms[i%numMessages].ShrinkTo(OutMessageHeaderSize)
 		}
 
 		b.SetBytes(int64(MaxReadSize))