Btrfs: extent_io and extent_state optimizations

The end_bio routines are changed to take a pointer to the extent state
struct, and the state tree is walked in order to set/clear appropriate
bits as IO completes.  This greatly reduces the number of rbtree searches
done by the end_bio handlers, and reduces lock contention.

The extent_io releasepage function is changed to avoid expensive searches
for locked state.
Signed-off-by: Chris Mason <chris.mason@oracle.com>

fs/btrfs/disk-io.c

@@ -274,7 +274,7 @@ int btree_readpage(struct file *file, struct page *page)
 	return extent_read_full_page(tree, page, btree_get_extent);
 }
 
-static int btree_releasepage(struct page *page, gfp_t unused_gfp_flags)
+static int btree_releasepage(struct page *page, gfp_t gfp_flags)
 {
 	struct extent_io_tree *tree;
 	struct extent_map_tree *map;
@@ -282,7 +282,7 @@ static int btree_releasepage(struct page *page, gfp_t unused_gfp_flags)
 
 	tree = &BTRFS_I(page->mapping->host)->io_tree;
 	map = &BTRFS_I(page->mapping->host)->extent_tree;
-	ret = try_release_extent_mapping(map, tree, page);
+	ret = try_release_extent_mapping(map, tree, page, gfp_flags);
 	if (ret == 1) {
 		ClearPagePrivate(page);
 		set_page_private(page, 0);

fs/btrfs/extent_io.h

@@ -23,19 +23,23 @@
 #define EXTENT_PAGE_PRIVATE 1
 #define EXTENT_PAGE_PRIVATE_FIRST_PAGE 3
 
+struct extent_state;
+
 struct extent_io_ops {
 	int (*fill_delalloc)(struct inode *inode, u64 start, u64 end);
 	int (*writepage_io_hook)(struct page *page, u64 start, u64 end);
 	int (*readpage_io_hook)(struct page *page, u64 start, u64 end);
-	int (*readpage_end_io_hook)(struct page *page, u64 start, u64 end);
-	void (*writepage_end_io_hook)(struct page *page, u64 start, u64 end);
+	int (*readpage_end_io_hook)(struct page *page, u64 start, u64 end,
+				    struct extent_state *state);
+	void (*writepage_end_io_hook)(struct page *page, u64 start, u64 end,
+				      struct extent_state *state);
 };
 
 struct extent_io_tree {
 	struct rb_root state;
 	struct address_space *mapping;
 	u64 dirty_bytes;
-	rwlock_t lock;
+	spinlock_t lock;
 	struct extent_io_ops *ops;
 	spinlock_t lru_lock;
 	struct list_head buffer_lru;
@@ -45,8 +49,8 @@ struct extent_io_tree {
 struct extent_state {
 	u64 start;
 	u64 end; /* inclusive */
-	int in_tree;
 	struct rb_node rb_node;
+	struct extent_io_tree *tree;
 	wait_queue_head_t wq;
 	atomic_t refs;
 	unsigned long state;
@@ -82,7 +86,8 @@ void extent_io_tree_init(struct extent_io_tree *tree,
 			  struct address_space *mapping, gfp_t mask);
 void extent_io_tree_empty_lru(struct extent_io_tree *tree);
 int try_release_extent_mapping(struct extent_map_tree *map,
-			       struct extent_io_tree *tree, struct page *page);
+			       struct extent_io_tree *tree, struct page *page,
+			       gfp_t mask);
 int lock_extent(struct extent_io_tree *tree, u64 start, u64 end, gfp_t mask);
 int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end, gfp_t mask);
 int extent_read_full_page(struct extent_io_tree *tree, struct page *page,

fs/btrfs/inode.c

@@ -331,7 +331,8 @@ int btrfs_readpage_io_hook(struct page *page, u64 start, u64 end)
 	return ret;
 }
 
-int btrfs_readpage_end_io_hook(struct page *page, u64 start, u64 end)
+int btrfs_readpage_end_io_hook(struct page *page, u64 start, u64 end,
+			       struct extent_state *state)
 {
 	size_t offset = start - ((u64)page->index << PAGE_CACHE_SHIFT);
 	struct inode *inode = page->mapping->host;
@@ -347,7 +348,12 @@ int btrfs_readpage_end_io_hook(struct page *page, u64 start, u64 end)
 	    btrfs_test_flag(inode, NODATASUM))
 		return 0;
 
-	ret = get_state_private(io_tree, start, &private);
+	if (state->start == start) {
+		private = state->private;
+		ret = 0;
+	} else {
+		ret = get_state_private(io_tree, start, &private);
+	}
 	local_irq_save(flags);
 	kaddr = kmap_atomic(page, KM_IRQ0);
 	if (ret) {
@@ -1830,7 +1836,7 @@ static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, int mode)
 }
 
 struct extent_map *btrfs_get_extent(struct inode *inode, struct page *page,
-				    size_t page_offset, u64 start, u64 len,
+				    size_t pg_offset, u64 start, u64 len,
 				    int create)
 {
 	int ret;
@@ -1865,7 +1871,10 @@ struct extent_map *btrfs_get_extent(struct inode *inode, struct page *page,
 			       start, len, em->start, em->len);
 			WARN_ON(1);
 		}
-		goto out;
+		if (em->block_start == EXTENT_MAP_INLINE && page)
+			free_extent_map(em);
+		else
+			goto out;
 	}
 	em = alloc_extent_map(GFP_NOFS);
 	if (!em) {
@@ -1930,6 +1939,7 @@ struct extent_map *btrfs_get_extent(struct inode *inode, struct page *page,
 		em->len = extent_end - extent_start;
 		goto insert;
 	} else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
+		u64 page_start;
 		unsigned long ptr;
 		char *map;
 		size_t size;
@@ -1959,16 +1969,17 @@ struct extent_map *btrfs_get_extent(struct inode *inode, struct page *page,
 			goto out;
 		}
 
-		extent_offset = ((u64)page->index << PAGE_CACHE_SHIFT) -
-			extent_start + page_offset;
-		copy_size = min_t(u64, PAGE_CACHE_SIZE - page_offset,
+		page_start = page_offset(page) + pg_offset;
+		extent_offset = page_start - extent_start;
+		copy_size = min_t(u64, PAGE_CACHE_SIZE - pg_offset,
 				size - extent_offset);
 		em->start = extent_start + extent_offset;
-		em->len = copy_size;
+		em->len = (copy_size + root->sectorsize - 1) &
+			~((u64)root->sectorsize - 1);
 		map = kmap(page);
 		ptr = btrfs_file_extent_inline_start(item) + extent_offset;
 		if (create == 0 && !PageUptodate(page)) {
-			read_extent_buffer(leaf, map + page_offset, ptr,
+			read_extent_buffer(leaf, map + pg_offset, ptr,
 					   copy_size);
 			flush_dcache_page(page);
 		} else if (create && PageUptodate(page)) {
@@ -1980,7 +1991,7 @@ struct extent_map *btrfs_get_extent(struct inode *inode, struct page *page,
 				trans = btrfs_start_transaction(root, 1);
 				goto again;
 			}
-			write_extent_buffer(leaf, map + page_offset, ptr,
+			write_extent_buffer(leaf, map + pg_offset, ptr,
 					    copy_size);
 			btrfs_mark_buffer_dirty(leaf);
 		}
@@ -2077,7 +2088,7 @@ btrfs_readpages(struct file *file, struct address_space *mapping,
 				btrfs_get_extent);
 }
 
-static int btrfs_releasepage(struct page *page, gfp_t unused_gfp_flags)
+static int btrfs_releasepage(struct page *page, gfp_t gfp_flags)
 {
 	struct extent_io_tree *tree;
 	struct extent_map_tree *map;
@@ -2085,7 +2096,7 @@ static int btrfs_releasepage(struct page *page, gfp_t unused_gfp_flags)
 
 	tree = &BTRFS_I(page->mapping->host)->io_tree;
 	map = &BTRFS_I(page->mapping->host)->extent_tree;
-	ret = try_release_extent_mapping(map, tree, page);
+	ret = try_release_extent_mapping(map, tree, page, gfp_flags);
 	if (ret == 1) {
 		ClearPagePrivate(page);
 		set_page_private(page, 0);