Add a C baseline for the benchmark.

README

@@ -39,7 +39,7 @@ EXAMPLES
     
 * fuse/loopback.go mounts another piece of the filesystem.
   Functionally, it is similar to a symlink.  A binary to run is in
-  example/loopback/ . For example
+i  example/loopback/ . For example
 
     mkdir /tmp/mountpoint
     example/loopback/loopback -debug /tmp/mountpoint /some/other/directory &
@@ -69,18 +69,18 @@ Tested on:
 
 BENCHMARKS
 
-We use threaded stats over a read-only filesystem for benchmarking. A
-script to do this is in example/benchmark.sh.  A libfuse baseline is
-given by running the same test against archivemount
-(http://www.cybernoia.de/software/archivemount/), with the locks in
-its GetAttr implementation removed.
+We use threaded stats over a read-only filesystem for benchmarking.
+Automated code is under benchmark/ .sh.  A simple C version of the
+same FS gives a FUSe baselin
 
-Data points (time per stat, Go-FUSE version May 1), using java 1.6
+Data points (time per stat, Go-FUSE version Sep 3), using java 1.6
 src.zip (7000 files).
 
 platform               libfuse      Go-FUSE     difference (%)
 
-Lenovo T60   (2cpu)    83us         99us        19%
+Lenovo T60   (2cpu)    106us        125us       18%
+
+(todo - revise timings below: )
 Lenovo T400  (2cpu)    38us         58us        52%
 DellT3500/Lucid (2cpu) 34us(*)      35us         3%
 DellT3500/Lucid (6cpu) 59us         76us        28%

all.bash

@@ -3,6 +3,7 @@ set -eux
 
 rm -f fuse/version.gen.go
 
+
 for target in "clean" "" "$@" ; do
   for d in fuse benchmark zipfs unionfs \
     example/hello example/loopback example/zipfs \
@@ -18,6 +19,7 @@ do
   (cd $d && gotest )
 done
 
+gomake -C benchmark cstatfs
 for d in benchmark
 do
   (cd $d && gotest -test.bench '.*' -test.cpu 1,2 )

benchmark/Makefile

@@ -6,3 +6,6 @@ TARG=github.com/hanwen/go-fuse/benchmark
 GOFILES=benchmark.go
 
 include $(GOROOT)/src/Make.pkg
+
+cstatfs: statfs.cc
+	g++ -Wall -std=c++0x `pkg-config fuse --cflags --libs` $< -o $@

benchmark/benchmark.go

@@ -4,6 +4,7 @@ package fuse
 
 import (
 	"fmt"
+	"log"
 	"math"
 	"os"
 	"regexp"
@@ -28,7 +29,10 @@ func BulkStat(parallelism int, files []string) float64 {
 				}
 
 				t := time.Nanoseconds()
-				os.Lstat(fn)
+				_, err := os.Lstat(fn)
+				if err != nil {
+					log.Fatal("All stats should succeed:", err)
+				}
 				dts <- time.Nanoseconds() - t
 			}
 		}()
@@ -77,9 +81,9 @@ func AnalyzeBenchmarkRuns(times []float64) {
 
 	fmt.Printf(
 		"%d samples\n"+
-			"avg %.2f ms 2sigma %.2f "+
-			"median %.2fms\n"+
-			"10%%tile %.2fms, 90%%tile %.2fms\n",
+			"avg %.3f ms 2sigma %.3f "+
+			"median %.3fms\n"+
+			"10%%tile %.3fms, 90%%tile %.3fms\n",
 		len(times), avg, 2*stddev, median, perc10, perc90)
 }
 

benchmark/stat_test.go

@@ -2,13 +2,15 @@ package fuse
 
 import (
 	"bufio"
+	"exec"
 	"fmt"
+	"github.com/hanwen/go-fuse/fuse"
 	"io/ioutil"
 	"log"
 	"os"
-	"github.com/hanwen/go-fuse/fuse"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -49,7 +51,6 @@ func (me *StatFs) GetAttr(name string, context *fuse.Context) (*os.FileInfo, fus
 }
 
 func (me *StatFs) OpenDir(name string, context *fuse.Context) (stream chan fuse.DirEntry, status fuse.Status) {
-	log.Printf("OPENDIR '%v', %v %v", name, me.entries, me.dirs)
 	entries := me.dirs[name]
 	if entries == nil {
 		return nil, fuse.ENOENT
@@ -128,18 +129,18 @@ func TestNewStatFs(t *testing.T) {
 	}
 }
 
-func BenchmarkThreadedStat(b *testing.B) {
-	b.StopTimer()
-	fs := NewStatFs()
+func GetTestLines() []string {
 	wd, _ := os.Getwd()
 	// Names from OpenJDK 1.6
-	f, err := os.Open(wd + "/testpaths.txt")
+	fn := wd + "/testpaths.txt"
+
+	f, err := os.Open(fn)
 	CheckSuccess(err)
 
 	defer f.Close()
 	r := bufio.NewReader(f)
 
-	files := []string{}
+	l := []string{}
 	for {
 		line, _, err := r.ReadLine()
 		if line == nil || err != nil {
@@ -147,16 +148,24 @@ func BenchmarkThreadedStat(b *testing.B) {
 		}
 
 		fn := string(line)
-		files = append(files, fn)
+		l = append(l, fn)
+	}
+	return l
+}
 
+func BenchmarkThreadedStat(b *testing.B) {
+	b.StopTimer()
+	fs := NewStatFs()
+	files := GetTestLines()
+	for _, fn := range files {
 		fs.add(fn, os.FileInfo{Mode: fuse.S_IFREG | 0644})
 	}
-	log.Printf("Read %d file names", len(files))
-
 	if len(files) == 0 {
 		log.Fatal("no files added")
 	}
 
+	log.Printf("Read %d file names", len(files))
+
 	ttl := 0.1
 	opts := fuse.FileSystemOptions{
 		EntryTimeout:    ttl,
@@ -197,3 +206,58 @@ func TestingBOnePass(b *testing.B, threads int, sleepTime float64, files []strin
 	}
 	return results
 }
+
+
+func BenchmarkCFuseThreadedStat(b *testing.B) {
+	log.Println("benchmarking CFuse")
+	
+	lines := GetTestLines()
+	unique := map[string]int{}
+	for _, l := range lines {
+		unique[l] = 1
+		dir, _ := filepath.Split(l)
+		for dir != "/" && dir != "" {
+			unique[dir] = 1
+			dir = filepath.Clean(dir)
+			dir, _ = filepath.Split(dir)
+		}			
+	}
+
+	out := []string{}
+	for k, _ := range unique {
+		out = append(out, k)
+	}
+
+	f, err := ioutil.TempFile("", "")
+	CheckSuccess(err)
+	sort.Strings(out)
+	for _, k := range out {
+		f.Write([]byte(fmt.Sprintf("/%s\n", k)))
+	}
+	f.Close()
+	
+	log.Println("Written:", f.Name())
+	mountPoint := fuse.MakeTempDir()
+	wd, _ := os.Getwd()
+	cmd := exec.Command(wd + "/cstatfs", mountPoint)
+	cmd.Env = append(os.Environ(), fmt.Sprintf("STATFS_INPUT=%s", f.Name()))
+	cmd.Start()
+
+	bin, err := exec.LookPath("fusermount")
+	CheckSuccess(err)
+	stop := exec.Command(bin, "-u", mountPoint)
+	CheckSuccess(err)
+	defer stop.Run()
+	
+	for i, l := range lines {
+		lines[i] = filepath.Join(mountPoint, l)
+	}
+
+	// Wait for the daemon to mount.
+	time.Sleep(0.2e9)
+	ttl := 1.0
+	log.Println("N = ", b.N)
+	threads := runtime.GOMAXPROCS(0)
+	results := TestingBOnePass(b, threads, ttl*1.2, lines)
+	AnalyzeBenchmarkRuns(results)	
+}

benchmark/statfs.cc

+//
+// g++ -Wall `pkg-config fuse --cflags --libs` statfs.cc -o statfs
+
+#include <unordered_map>
+#include <string>
+
+using std::string;
+using std::unordered_map;
+
+#define FUSE_USE_VERSION 26
+
+extern "C" {
+#include <fuse.h>
+}
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <fcntl.h>
+
+
+class StatFs {
+public:
+  void readFrom(const string& fn);
+  unordered_map<string, bool> is_dir_;
+
+  int GetAttr(const char *name, struct stat *statbuf) {
+    if (strcmp(name, "/") == 0) {
+      statbuf->st_mode = S_IFDIR | 0777;
+      return 0;
+    }
+    unordered_map<string, bool>::const_iterator it(is_dir_.find(name));
+    if (it == is_dir_.end()) {
+      return -ENOENT;
+    }
+
+    if (it->second) {
+      statbuf->st_mode = S_IFDIR | 0777; 
+   } else {
+      statbuf->st_nlink = 1;
+      statbuf->st_mode = S_IFREG | 0666;
+    }
+
+    return 0;
+  }
+  
+};
+
+StatFs *global;
+int global_getattr(const char *name, struct stat *statbuf) {
+  return global->GetAttr(name, statbuf);
+}
+
+void StatFs::readFrom(const string& fn) {
+  FILE *f = fopen(fn.c_str(), "r");
+
+  char line[1024];
+  while (char *s = fgets(line, sizeof(line), f)) {
+    int l = strlen(s);
+    if (line[l-1] == '\n') {
+      line[l-1] = '\0';
+      l--;
+    }
+    bool is_dir = line[l-1] == '/';
+    if (is_dir) {
+      line[l-1] = '\0';
+    }
+    is_dir_[line] = is_dir;
+  }
+  fclose(f);
+}
+
+int main(int argc, char *argv[])
+{
+  global = new StatFs;
+  // don't want to know about fuselib's option handling
+  char *in = getenv("STATFS_INPUT");
+  if (!in || !*in) {
+    fprintf(stderr, "pass file in $STATFS_INPUT\n");
+    exit(2);
+  }
+      
+    global->readFrom(in);
+  struct fuse_operations statfs_oper  = {0};
+  statfs_oper.getattr = &global_getattr;
+  return fuse_main(argc, argv, &statfs_oper, NULL);
+}