exofs: ios: Move to a per inode components & device-table

Exofs raid engine was saving on memory space by having a single layout-info,
single pid, and a single device-table, global to the filesystem. Then passing
a credential and object_id info at the io_state level, private for each
inode. It would also devise this contraption of rotating the device table
view for each inode->ino to spread out the device usage.

This is not compatible with the pnfs-objects standard, demanding that
each inode can have it's own layout-info, device-table, and each object
component it's own pid, oid and creds.

So: Bring exofs raid engine to be usable for generic pnfs-objects use by:

* Define an exofs_comp structure that holds obj_id and credential info.

* Break up exofs_layout struct to an exofs_components structure that holds a
  possible array of exofs_comp and the array of devices + the size of the
  arrays.

* Add a "comps" parameter to get_io_state() that specifies the ids creds
  and device array to use for each IO.

  This enables to keep the layout global, but the device-table view, creds
  and IDs at the inode level. It only adds two 64bit to each inode, since
  some of these members already existed in another form.

* ios raid engine now access layout-info and comps-info through the passed
  pointers. Everything is pre-prepared by caller for generic access of
  these structures and arrays.

At the exofs Level:

* Super block holds an exofs_components struct that holds the device
  array, previously in layout. The devices there are in device-table
  order. The device-array is twice bigger and repeats the device-table
  twice so now each inode's device array can point to a random device
  and have a round-robin view of the table, making it compatible to
  previous exofs versions.

* Each inode has an exofs_components struct that is initialized at
  load time, with it's own view of the device table IDs and creds.
  When doing IO this gets passed to the io_state together with the
  layout.

While preforming this change. Bugs where found where credentials with the
wrong IDs where used to access the different SB objects (super.c). As well
as some dead code. It was never noticed because the target we use does not
check the credentials.
Signed-off-by: Boaz Harrosh <bharrosh@panasas.com>

fs/exofs/exofs.h

@@ -52,9 +52,12 @@
 /* u64 has problems with printk this will cast it to unsigned long long */
 #define _LLU(x) (unsigned long long)(x)
 
-struct exofs_layout {
-	osd_id		s_pid;			/* partition ID of file system*/
+struct exofs_comp {
+	struct osd_obj_id	obj;
+	u8			cred[OSD_CAP_LEN];
+};
 
+struct exofs_layout {
 	/* Our way of looking at the data_map */
 	unsigned stripe_unit;
 	unsigned mirrors_p1;
@@ -62,11 +65,18 @@ struct exofs_layout {
 	unsigned group_width;
 	u64	 group_depth;
 	unsigned group_count;
+};
 
-	enum exofs_inode_layout_gen_functions lay_func;
-
-	unsigned	s_numdevs;		/* Num of devices in array    */
-	struct osd_dev	**s_ods;		/* osd_dev array              */
+struct exofs_components {
+	unsigned	numdevs;		/* Num of devices in array    */
+	/* If @single_comp == EC_SINGLE_COMP, @comps points to a single
+	 * component. else there are @numdevs components
+	 */
+	enum EC_COMP_USAGE {
+		EC_SINGLE_COMP = 0, EC_MULTPLE_COMPS = 0xffffffff
+	} single_comp;
+	struct exofs_comp *comps;
+	struct osd_dev	**ods;			/* osd_dev array              */
 };
 
 /*
@@ -81,12 +91,13 @@ struct exofs_sb_info {
 	spinlock_t	s_next_gen_lock;	/* spinlock for gen # update  */
 	u32		s_next_generation;	/* next gen # to use          */
 	atomic_t	s_curr_pending;		/* number of pending commands */
-	uint8_t		s_cred[OSD_CAP_LEN];	/* credential for the fscb    */
 
 	struct pnfs_osd_data_map data_map;	/* Default raid to use
 						 * FIXME: Needed ?
 						 */
 	struct exofs_layout	layout;		/* Default files layout       */
+	struct exofs_comp one_comp;		/* id & cred of partition id=0*/
+	struct exofs_components comps;		/* comps for the partition    */
 	struct osd_dev	*_min_one_dev[1];	/* Place holder for one dev   */
 };
 
@@ -100,7 +111,8 @@ struct exofs_i_info {
 	uint32_t       i_data[EXOFS_IDATA];/*short symlink names and device #s*/
 	uint32_t       i_dir_start_lookup; /* which page to start lookup      */
 	uint64_t       i_commit_size;      /* the object's written length     */
-	uint8_t        i_cred[OSD_CAP_LEN];/* all-powerful credential         */
+	struct exofs_comp one_comp;	   /* same component for all devices  */
+	struct exofs_components comps;	   /* inode view of the device table  */
 };
 
 static inline osd_id exofs_oi_objno(struct exofs_i_info *oi)
@@ -118,8 +130,7 @@ struct exofs_io_state {
 	exofs_io_done_fn	done;
 
 	struct exofs_layout	*layout;
-	struct osd_obj_id	obj;
-	u8			*cred;
+	struct exofs_components	*comps;
 
 	/* Global read/write IO*/
 	loff_t			offset;
@@ -199,20 +210,6 @@ static inline struct exofs_i_info *exofs_i(struct inode *inode)
 	return container_of(inode, struct exofs_i_info, vfs_inode);
 }
 
-/*
- * Given a layout, object_number and stripe_index return the associated global
- * dev_index
- */
-unsigned exofs_layout_od_id(struct exofs_layout *layout,
-			    osd_id obj_no, unsigned layout_index);
-
-static inline struct osd_dev *exofs_ios_od(struct exofs_io_state *ios,
-					   unsigned layout_index)
-{
-	return ios->layout->s_ods[
-		exofs_layout_od_id(ios->layout, ios->obj.id, layout_index)];
-}
-
 /*
  * Maximum count of links to a file
  */
@@ -223,9 +220,12 @@ static inline struct osd_dev *exofs_ios_od(struct exofs_io_state *ios,
  *************************/
 
 /* ios.c */
-int  exofs_get_rw_state(struct exofs_layout *layout, bool is_reading,
-			u64 offset, u64 length, struct exofs_io_state **ios);
+int exofs_get_rw_state(struct exofs_layout *layout,
+		       struct exofs_components *comps,
+		       bool is_reading, u64 offset, u64 length,
+		       struct exofs_io_state **ios);
 int exofs_get_io_state(struct exofs_layout *layout,
+		       struct exofs_components *comps,
 		       struct exofs_io_state **ios);
 void exofs_put_io_state(struct exofs_io_state *ios);
 
@@ -235,27 +235,12 @@ int exofs_sbi_create(struct exofs_io_state *ios);
 int exofs_sbi_remove(struct exofs_io_state *ios);
 int exofs_sbi_write(struct exofs_io_state *ios);
 int exofs_sbi_read(struct exofs_io_state *ios);
+int exofs_truncate(struct exofs_layout *layout, struct exofs_components *comps,
+		 u64 size);
 
 int extract_attr_from_ios(struct exofs_io_state *ios, struct osd_attr *attr);
 extern const struct osd_attr g_attr_logical_length;
 
-int exofs_oi_truncate(struct exofs_i_info *oi, u64 new_len);
-static inline int exofs_oi_write(struct exofs_i_info *oi,
-				 struct exofs_io_state *ios)
-{
-	ios->obj.id = exofs_oi_objno(oi);
-	ios->cred = oi->i_cred;
-	return exofs_sbi_write(ios);
-}
-
-static inline int exofs_oi_read(struct exofs_i_info *oi,
-				struct exofs_io_state *ios)
-{
-	ios->obj.id = exofs_oi_objno(oi);
-	ios->cred = oi->i_cred;
-	return exofs_sbi_read(ios);
-}
-
 /* inode.c               */
 unsigned exofs_max_io_pages(struct exofs_layout *layout,
 			    unsigned expected_pages);
@@ -307,4 +292,33 @@ extern const struct inode_operations exofs_special_inode_operations;
 extern const struct inode_operations exofs_symlink_inode_operations;
 extern const struct inode_operations exofs_fast_symlink_inode_operations;
 
+/* exofs_init_comps will initialize an exofs_components device array
+ * pointing to a single exofs_comp struct, and a round-robin view
+ * of the device table.
+ * The first device of each inode is the [inode->ino % num_devices]
+ * and the rest of the devices sequentially following where the
+ * first device is after the last device.
+ * It is assumed that the global device array at @sbi is twice
+ * bigger and that the device table repeats twice.
+ * See: exofs_read_lookup_dev_table()
+ */
+static inline void exofs_init_comps(struct exofs_components *comps,
+				    struct exofs_comp *one_comp,
+				    struct exofs_sb_info *sbi, osd_id oid)
+{
+	unsigned dev_mod = (unsigned)oid, first_dev;
+
+	one_comp->obj.partition = sbi->one_comp.obj.partition;
+	one_comp->obj.id = oid;
+	exofs_make_credential(one_comp->cred, &one_comp->obj);
+
+	comps->numdevs = sbi->comps.numdevs;
+	comps->single_comp = EC_SINGLE_COMP;
+	comps->comps = one_comp;
+
+	/* Round robin device view of the table */
+	first_dev = (dev_mod * sbi->layout.mirrors_p1) % sbi->comps.numdevs;
+	comps->ods = sbi->comps.ods + first_dev;
+}
+
 #endif

fs/exofs/inode.c

@@ -270,7 +270,7 @@ static int read_exec(struct page_collect *pcol)
 		return 0;
 
 	if (!pcol->ios) {
-		int ret = exofs_get_rw_state(&pcol->sbi->layout, true,
+		int ret = exofs_get_rw_state(&pcol->sbi->layout, &oi->comps, true,
 					     pcol->pg_first << PAGE_CACHE_SHIFT,
 					     pcol->length, &pcol->ios);
 
@@ -283,7 +283,7 @@ static int read_exec(struct page_collect *pcol)
 	ios->nr_pages = pcol->nr_pages;
 
 	if (pcol->read_4_write) {
-		exofs_oi_read(oi, pcol->ios);
+		exofs_sbi_read(pcol->ios);
 		return __readpages_done(pcol);
 	}
 
@@ -296,14 +296,14 @@ static int read_exec(struct page_collect *pcol)
 	*pcol_copy = *pcol;
 	ios->done = readpages_done;
 	ios->private = pcol_copy;
-	ret = exofs_oi_read(oi, ios);
+	ret = exofs_sbi_read(ios);
 	if (unlikely(ret))
 		goto err;
 
 	atomic_inc(&pcol->sbi->s_curr_pending);
 
 	EXOFS_DBGMSG2("read_exec obj=0x%llx start=0x%llx length=0x%lx\n",
-		  ios->obj.id, _LLU(ios->offset), pcol->length);
+		  oi->one_comp.obj.id, _LLU(ios->offset), pcol->length);
 
 	/* pages ownership was passed to pcol_copy */
 	_pcol_reset(pcol);
@@ -516,7 +516,7 @@ static int write_exec(struct page_collect *pcol)
 		return 0;
 
 	BUG_ON(pcol->ios);
-	ret = exofs_get_rw_state(&pcol->sbi->layout, false,
+	ret = exofs_get_rw_state(&pcol->sbi->layout, &oi->comps, false,
 				 pcol->pg_first << PAGE_CACHE_SHIFT,
 				 pcol->length, &pcol->ios);
 
@@ -538,9 +538,9 @@ static int write_exec(struct page_collect *pcol)
 	ios->done = writepages_done;
 	ios->private = pcol_copy;
 
-	ret = exofs_oi_write(oi, ios);
+	ret = exofs_sbi_write(ios);
 	if (unlikely(ret)) {
-		EXOFS_ERR("write_exec: exofs_oi_write() Failed\n");
+		EXOFS_ERR("write_exec: exofs_sbi_write() Failed\n");
 		goto err;
 	}
 
@@ -855,11 +855,12 @@ static inline int exofs_inode_is_fast_symlink(struct inode *inode)
 static int _do_truncate(struct inode *inode, loff_t newsize)
 {
 	struct exofs_i_info *oi = exofs_i(inode);
+	struct exofs_sb_info *sbi = inode->i_sb->s_fs_info;
 	int ret;
 
 	inode->i_mtime = inode->i_ctime = CURRENT_TIME;
 
-	ret = exofs_oi_truncate(oi, (u64)newsize);
+	ret = exofs_truncate(&sbi->layout, &oi->comps, (u64)newsize);
 	if (likely(!ret))
 		truncate_setsize(inode, newsize);
 
@@ -926,18 +927,14 @@ static int exofs_get_inode(struct super_block *sb, struct exofs_i_info *oi,
 	struct exofs_on_disk_inode_layout *layout;
 	int ret;
 
-	ret = exofs_get_io_state(&sbi->layout, &ios);
+	ret = exofs_get_io_state(&sbi->layout, &oi->comps, &ios);
 	if (unlikely(ret)) {
 		EXOFS_ERR("%s: exofs_get_io_state failed.\n", __func__);
 		return ret;
 	}
 
-	ios->obj.id = exofs_oi_objno(oi);
-	exofs_make_credential(oi->i_cred, &ios->obj);
-	ios->cred = oi->i_cred;
-
-	attrs[1].len = exofs_on_disk_inode_layout_size(sbi->layout.s_numdevs);
-	attrs[2].len = exofs_on_disk_inode_layout_size(sbi->layout.s_numdevs);
+	attrs[1].len = exofs_on_disk_inode_layout_size(sbi->comps.numdevs);
+	attrs[2].len = exofs_on_disk_inode_layout_size(sbi->comps.numdevs);
 
 	ios->in_attr = attrs;
 	ios->in_attr_len = ARRAY_SIZE(attrs);
@@ -945,7 +942,7 @@ static int exofs_get_inode(struct super_block *sb, struct exofs_i_info *oi,
 	ret = exofs_sbi_read(ios);
 	if (unlikely(ret)) {
 		EXOFS_ERR("object(0x%llx) corrupted, return empty file=>%d\n",
-			  _LLU(ios->obj.id), ret);
+			  _LLU(oi->one_comp.obj.id), ret);
 		memset(inode, 0, sizeof(*inode));
 		inode->i_mode = 0040000 | (0777 & ~022);
 		/* If object is lost on target we might as well enable it's
@@ -1021,6 +1018,8 @@ struct inode *exofs_iget(struct super_block *sb, unsigned long ino)
 		return inode;
 	oi = exofs_i(inode);
 	__oi_init(oi);
+	exofs_init_comps(&oi->comps, &oi->one_comp, sb->s_fs_info,
+			 exofs_oi_objno(oi));
 
 	/* read the inode from the osd */
 	ret = exofs_get_inode(sb, oi, &fcb);
@@ -1126,7 +1125,8 @@ static void create_done(struct exofs_io_state *ios, void *p)
 
 	if (unlikely(ret)) {
 		EXOFS_ERR("object=0x%llx creation failed in pid=0x%llx",
-			  _LLU(exofs_oi_objno(oi)), _LLU(sbi->layout.s_pid));
+			  _LLU(exofs_oi_objno(oi)),
+			  _LLU(oi->one_comp.obj.partition));
 		/*TODO: When FS is corrupted creation can fail, object already
 		 * exist. Get rid of this asynchronous creation, if exist
 		 * increment the obj counter and try the next object. Until we
@@ -1145,14 +1145,13 @@ static void create_done(struct exofs_io_state *ios, void *p)
  */
 struct inode *exofs_new_inode(struct inode *dir, int mode)
 {
-	struct super_block *sb;
+	struct super_block *sb = dir->i_sb;
+	struct exofs_sb_info *sbi = sb->s_fs_info;
 	struct inode *inode;
 	struct exofs_i_info *oi;
-	struct exofs_sb_info *sbi;
 	struct exofs_io_state *ios;
 	int ret;
 
-	sb = dir->i_sb;
 	inode = new_inode(sb);
 	if (!inode)
 		return ERR_PTR(-ENOMEM);
@@ -1162,8 +1161,6 @@ struct inode *exofs_new_inode(struct inode *dir, int mode)
 
 	set_obj_2bcreated(oi);
 
-	sbi = sb->s_fs_info;
-
 	inode->i_mapping->backing_dev_info = sb->s_bdi;
 	inode_init_owner(inode, dir, mode);
 	inode->i_ino = sbi->s_nextid++;
@@ -1175,22 +1172,21 @@ struct inode *exofs_new_inode(struct inode *dir, int mode)
 	spin_unlock(&sbi->s_next_gen_lock);
 	insert_inode_hash(inode);
 
+	exofs_init_comps(&oi->comps, &oi->one_comp, sb->s_fs_info,
+			 exofs_oi_objno(oi));
 	exofs_sbi_write_stats(sbi); /* Make sure new sbi->s_nextid is on disk */
 
 	mark_inode_dirty(inode);
 
-	ret = exofs_get_io_state(&sbi->layout, &ios);
+	ret = exofs_get_io_state(&sbi->layout, &oi->comps, &ios);
 	if (unlikely(ret)) {
 		EXOFS_ERR("exofs_new_inode: exofs_get_io_state failed\n");
 		return ERR_PTR(ret);
 	}
 
-	ios->obj.id = exofs_oi_objno(oi);
-	exofs_make_credential(oi->i_cred, &ios->obj);
-
 	ios->done = create_done;
 	ios->private = inode;
-	ios->cred = oi->i_cred;
+
 	ret = exofs_sbi_create(ios);
 	if (ret) {
 		exofs_put_io_state(ios);
@@ -1271,7 +1267,7 @@ static int exofs_update_inode(struct inode *inode, int do_sync)
 	} else
 		memcpy(fcb->i_data, oi->i_data, sizeof(fcb->i_data));
 
-	ret = exofs_get_io_state(&sbi->layout, &ios);
+	ret = exofs_get_io_state(&sbi->layout, &oi->comps, &ios);
 	if (unlikely(ret)) {
 		EXOFS_ERR("%s: exofs_get_io_state failed.\n", __func__);
 		goto free_args;
@@ -1290,7 +1286,7 @@ static int exofs_update_inode(struct inode *inode, int do_sync)
 		ios->private = args;
 	}
 
-	ret = exofs_oi_write(oi, ios);
+	ret = exofs_sbi_write(ios);
 	if (!do_sync && !ret) {
 		atomic_inc(&sbi->s_curr_pending);
 		goto out; /* deallocation in updatei_done */
@@ -1354,16 +1350,15 @@ void exofs_evict_inode(struct inode *inode)
 	/* ignore the error, attempt a remove anyway */
 
 	/* Now Remove the OSD objects */
-	ret = exofs_get_io_state(&sbi->layout, &ios);
+	ret = exofs_get_io_state(&sbi->layout, &oi->comps, &ios);
 	if (unlikely(ret)) {
 		EXOFS_ERR("%s: exofs_get_io_state failed\n", __func__);
 		return;
 	}
 
-	ios->obj.id = exofs_oi_objno(oi);
 	ios->done = delete_done;
 	ios->private = sbi;
-	ios->cred = oi->i_cred;
+
 	ret = exofs_sbi_remove(ios);
 	if (ret) {
 		EXOFS_ERR("%s: exofs_sbi_remove failed\n", __func__);

fs/exofs/ios.c

@@ -31,24 +31,41 @@
 #define EXOFS_DBGMSG2(M...) do {} while (0)
 /* #define EXOFS_DBGMSG2 EXOFS_DBGMSG */
 
-int  exofs_get_rw_state(struct exofs_layout *layout, bool is_reading,
-			u64 offset, u64 length, struct exofs_io_state **pios)
+static u8 *_ios_cred(struct exofs_io_state *ios, unsigned index)
+{
+	return ios->comps->comps[index & ios->comps->single_comp].cred;
+}
+
+static struct osd_obj_id *_ios_obj(struct exofs_io_state *ios, unsigned index)
+{
+	return &ios->comps->comps[index & ios->comps->single_comp].obj;
+}
+
+static struct osd_dev *_ios_od(struct exofs_io_state *ios, unsigned index)
+{
+	return ios->comps->ods[index];
+}
+
+int exofs_get_rw_state(struct exofs_layout *layout,
+			struct exofs_components *comps,
+			bool is_reading, u64 offset, u64 length,
+			struct exofs_io_state **pios)
 {
 	struct exofs_io_state *ios;
 
 	/*TODO: Maybe use kmem_cach per sbi of size
 	 * exofs_io_state_size(layout->s_numdevs)
 	 */
-	ios = kzalloc(exofs_io_state_size(layout->s_numdevs), GFP_KERNEL);
+	ios = kzalloc(exofs_io_state_size(comps->numdevs), GFP_KERNEL);
 	if (unlikely(!ios)) {
 		EXOFS_DBGMSG("Failed kzalloc bytes=%d\n",
-			     exofs_io_state_size(layout->s_numdevs));
+			     exofs_io_state_size(comps->numdevs));
 		*pios = NULL;
 		return -ENOMEM;
 	}
 
 	ios->layout = layout;
-	ios->obj.partition = layout->s_pid;
+	ios->comps = comps;
 	ios->offset = offset;
 	ios->length = length;
 	ios->reading = is_reading;
@@ -58,9 +75,10 @@ int  exofs_get_rw_state(struct exofs_layout *layout, bool is_reading,
 }
 
 int  exofs_get_io_state(struct exofs_layout *layout,
+			struct exofs_components *comps,
 			struct exofs_io_state **ios)
 {
-	return exofs_get_rw_state(layout, true, 0, 0, ios);
+	return exofs_get_rw_state(layout, comps, true, 0, 0, ios);
 }
 
 void exofs_put_io_state(struct exofs_io_state *ios)
@@ -119,7 +137,7 @@ static int exofs_io_execute(struct exofs_io_state *ios)
 		if (unlikely(!or))
 			continue;
 
-		ret = osd_finalize_request(or, 0, ios->cred, NULL);
+		ret = osd_finalize_request(or, 0, _ios_cred(ios, i), NULL);
 		if (unlikely(ret)) {
 			EXOFS_DBGMSG("Failed to osd_finalize_request() => %d\n",
 				     ret);
@@ -300,7 +318,7 @@ static int _add_stripe_unit(struct exofs_io_state *ios,  unsigned *cur_pg,
 {
 	unsigned pg = *cur_pg;
 	struct request_queue *q =
-			osd_request_queue(exofs_ios_od(ios, per_dev->dev));
+			osd_request_queue(_ios_od(ios, per_dev->dev));
 
 	per_dev->length += cur_len;
 
@@ -440,10 +458,10 @@ int exofs_sbi_create(struct exofs_io_state *ios)
 {
 	int i, ret;
 
-	for (i = 0; i < ios->layout->s_numdevs; i++) {
+	for (i = 0; i < ios->comps->numdevs; i++) {
 		struct osd_request *or;
 
-		or = osd_start_request(exofs_ios_od(ios, i), GFP_KERNEL);
+		or = osd_start_request(_ios_od(ios, i), GFP_KERNEL);
 		if (unlikely(!or)) {
 			EXOFS_ERR("%s: osd_start_request failed\n", __func__);
 			ret = -ENOMEM;
@@ -452,7 +470,7 @@ int exofs_sbi_create(struct exofs_io_state *ios)
 		ios->per_dev[i].or = or;
 		ios->numdevs++;
 
-		osd_req_create_object(or, &ios->obj);
+		osd_req_create_object(or, _ios_obj(ios, i));
 	}
 	ret = exofs_io_execute(ios);
 
@@ -464,10 +482,10 @@ int exofs_sbi_remove(struct exofs_io_state *ios)
 {
 	int i, ret;
 
-	for (i = 0; i < ios->layout->s_numdevs; i++) {
+	for (i = 0; i < ios->comps->numdevs; i++) {
 		struct osd_request *or;
 
-		or = osd_start_request(exofs_ios_od(ios, i), GFP_KERNEL);
+		or = osd_start_request(_ios_od(ios, i), GFP_KERNEL);
 		if (unlikely(!or)) {
 			EXOFS_ERR("%s: osd_start_request failed\n", __func__);
 			ret = -ENOMEM;
@@ -476,7 +494,7 @@ int exofs_sbi_remove(struct exofs_io_state *ios)
 		ios->per_dev[i].or = or;
 		ios->numdevs++;
 
-		osd_req_remove_object(or, &ios->obj);
+		osd_req_remove_object(or, _ios_obj(ios, i));
 	}
 	ret = exofs_io_execute(ios);
 
@@ -498,7 +516,7 @@ static int _sbi_write_mirror(struct exofs_io_state *ios, int cur_comp)
 		struct exofs_per_dev_state *per_dev = &ios->per_dev[cur_comp];
 		struct osd_request *or;
 
-		or = osd_start_request(exofs_ios_od(ios, dev), GFP_KERNEL);
+		or = osd_start_request(_ios_od(ios, dev), GFP_KERNEL);
 		if (unlikely(!or)) {
 			EXOFS_ERR("%s: osd_start_request failed\n", __func__);
 			ret = -ENOMEM;
@@ -533,25 +551,29 @@ static int _sbi_write_mirror(struct exofs_io_state *ios, int cur_comp)
 				bio->bi_rw |= REQ_WRITE;
 			}
 
-			osd_req_write(or, &ios->obj, per_dev->offset, bio,
-				      per_dev->length);
+			osd_req_write(or, _ios_obj(ios, dev), per_dev->offset,
+				      bio, per_dev->length);
 			EXOFS_DBGMSG("write(0x%llx) offset=0x%llx "
 				      "length=0x%llx dev=%d\n",
-				     _LLU(ios->obj.id), _LLU(per_dev->offset),
+				     _LLU(_ios_obj(ios, dev)->id),
+				     _LLU(per_dev->offset),
 				     _LLU(per_dev->length), dev);
 		} else if (ios->kern_buff) {
-			ret = osd_req_write_kern(or, &ios->obj, per_dev->offset,
+			ret = osd_req_write_kern(or, _ios_obj(ios, dev),
+						 per_dev->offset,
 						 ios->kern_buff, ios->length);
 			if (unlikely(ret))
 				goto out;
 			EXOFS_DBGMSG2("write_kern(0x%llx) offset=0x%llx "
 				      "length=0x%llx dev=%d\n",
-				     _LLU(ios->obj.id), _LLU(per_dev->offset),
+				     _LLU(_ios_obj(ios, dev)->id),
+				     _LLU(per_dev->offset),
 				     _LLU(ios->length), dev);
 		} else {
-			osd_req_set_attributes(or, &ios->obj);
+			osd_req_set_attributes(or, _ios_obj(ios, dev));
 			EXOFS_DBGMSG2("obj(0x%llx) set_attributes=%d dev=%d\n",
-				     _LLU(ios->obj.id), ios->out_attr_len, dev);
+				     _LLU(_ios_obj(ios, dev)->id),
+				     ios->out_attr_len, dev);
 		}
 
 		if (ios->out_attr)
@@ -590,13 +612,14 @@ static int _sbi_read_mirror(struct exofs_io_state *ios, unsigned cur_comp)
 {
 	struct osd_request *or;
 	struct exofs_per_dev_state *per_dev = &ios->per_dev[cur_comp];
-	unsigned first_dev = (unsigned)ios->obj.id;
+	struct osd_obj_id *obj = _ios_obj(ios, cur_comp);
+	unsigned first_dev = (unsigned)obj->id;
 
 	if (ios->pages && !per_dev->length)
 		return 0; /* Just an empty slot */
 
 	first_dev = per_dev->dev + first_dev % ios->layout->mirrors_p1;
-	or = osd_start_request(exofs_ios_od(ios, first_dev), GFP_KERNEL);
+	or = osd_start_request(_ios_od(ios, first_dev), GFP_KERNEL);
 	if (unlikely(!or)) {
 		EXOFS_ERR("%s: osd_start_request failed\n", __func__);
 		return -ENOMEM;
@@ -604,25 +627,26 @@ static int _sbi_read_mirror(struct exofs_io_state *ios, unsigned cur_comp)
 	per_dev->or = or;
 
 	if (ios->pages) {
-		osd_req_read(or, &ios->obj, per_dev->offset,
+		osd_req_read(or, obj, per_dev->offset,
 				per_dev->bio, per_dev->length);
 		EXOFS_DBGMSG("read(0x%llx) offset=0x%llx length=0x%llx"
-			     " dev=%d\n", _LLU(ios->obj.id),
+			     " dev=%d\n", _LLU(obj->id),
 			     _LLU(per_dev->offset), _LLU(per_dev->length),
 			     first_dev);
 	} else if (ios->kern_buff) {
-		int ret = osd_req_read_kern(or, &ios->obj, per_dev->offset,
+		int ret = osd_req_read_kern(or, obj, per_dev->offset,
 					    ios->kern_buff, ios->length);
 		EXOFS_DBGMSG2("read_kern(0x%llx) offset=0x%llx "
 			      "length=0x%llx dev=%d ret=>%d\n",
-			      _LLU(ios->obj.id), _LLU(per_dev->offset),
+			      _LLU(obj->id), _LLU(per_dev->offset),
 			      _LLU(ios->length), first_dev, ret);
 		if (unlikely(ret))
 			return ret;
 	} else {
-		osd_req_get_attributes(or, &ios->obj);
+		osd_req_get_attributes(or, obj);
 		EXOFS_DBGMSG2("obj(0x%llx) get_attributes=%d dev=%d\n",
-			      _LLU(ios->obj.id), ios->in_attr_len, first_dev);
+			      _LLU(obj->id),
+			      ios->in_attr_len, first_dev);
 	}
 	if (ios->out_attr)
 		osd_req_add_set_attr_list(or, ios->out_attr, ios->out_attr_len);
@@ -682,14 +706,14 @@ static int _truncate_mirrors(struct exofs_io_state *ios, unsigned cur_comp,
 		struct exofs_per_dev_state *per_dev = &ios->per_dev[cur_comp];
 		struct osd_request *or;
 
-		or = osd_start_request(exofs_ios_od(ios, cur_comp), GFP_KERNEL);
+		or = osd_start_request(_ios_od(ios, cur_comp), GFP_KERNEL);
 		if (unlikely(!or)) {
 			EXOFS_ERR("%s: osd_start_request failed\n", __func__);
 			return -ENOMEM;
 		}
 		per_dev->or = or;
 
-		osd_req_set_attributes(or, &ios->obj);
+		osd_req_set_attributes(or, _ios_obj(ios, cur_comp));
 		osd_req_add_set_attr_list(or, attr, 1);
 	}
 
@@ -721,9 +745,9 @@ void _calc_trunk_info(struct exofs_layout *layout, u64 file_offset,
 	ti->max_devs = layout->group_width * layout->group_count;
 }
 
-int exofs_oi_truncate(struct exofs_i_info *oi, u64 size)
+int exofs_truncate(struct exofs_layout *layout, struct exofs_components *comps,
+		   u64 size)
 {
-	struct exofs_sb_info *sbi = oi->vfs_inode.i_sb->s_fs_info;
 	struct exofs_io_state *ios;
 	struct exofs_trunc_attr {
 		struct osd_attr attr;
@@ -732,7 +756,7 @@ int exofs_oi_truncate(struct exofs_i_info *oi, u64 size)
 	struct _trunc_info ti;
 	int i, ret;
 
-	ret = exofs_get_io_state(&sbi->layout, &ios);
+	ret = exofs_get_io_state(layout, comps, &ios);
 	if (unlikely(ret))
 		return ret;
 
@@ -745,9 +769,7 @@ int exofs_oi_truncate(struct exofs_i_info *oi, u64 size)
 		goto out;
 	}
 
-	ios->obj.id = exofs_oi_objno(oi);
-	ios->cred = oi->i_cred;
-	ios->numdevs = ios->layout->s_numdevs;
+	ios->numdevs = ios->comps->numdevs;
 
 	for (i = 0; i < ti.max_devs; ++i) {
 		struct exofs_trunc_attr *size_attr = &size_attrs[i];
@@ -770,7 +792,7 @@ int exofs_oi_truncate(struct exofs_i_info *oi, u64 size)
 		size_attr->attr.val_ptr = &size_attr->newsize;
 
 		EXOFS_DBGMSG("trunc(0x%llx) obj_offset=0x%llx dev=%d\n",
-			     _LLU(ios->obj.id), _LLU(obj_size), i);
+			     _LLU(comps->comps->obj.id), _LLU(obj_size), i);
 		ret = _truncate_mirrors(ios, i * ios->layout->mirrors_p1,
 					&size_attr->attr);
 		if (unlikely(ret))

fs/exofs/super.c

@@ -253,22 +253,6 @@ static int exofs_read_kern(struct osd_dev *od, u8 *cred, struct osd_obj_id *obj,
 	return ret;
 }
 
-unsigned exofs_layout_od_id(struct exofs_layout *layout,
-			    osd_id obj_no, unsigned layout_index)
-{
-/*	switch (layout->lay_func) {
-	case LAYOUT_MOVING_WINDOW:
-	{*/
-		unsigned dev_mod = obj_no;
-
-		return (layout_index + dev_mod * layout->mirrors_p1) %
-							      layout->s_numdevs;
-/*	}
-	case LAYOUT_FUNC_IMPLICT:
-		return layout->devs[layout_index];
-	}*/
-}
-
 static const struct osd_attr g_attr_sb_stats = ATTR_DEF(
 	EXOFS_APAGE_SB_DATA,
 	EXOFS_ATTR_SB_STATS,
@@ -282,14 +266,12 @@ static int __sbi_read_stats(struct exofs_sb_info *sbi)
 	struct exofs_io_state *ios;
 	int ret;
 
-	ret = exofs_get_io_state(&sbi->layout, &ios);
+	ret = exofs_get_io_state(&sbi->layout, &sbi->comps, &ios);
 	if (unlikely(ret)) {
 		EXOFS_ERR("%s: exofs_get_io_state failed.\n", __func__);
 		return ret;
 	}
 
-	ios->cred = sbi->s_cred;
-
 	ios->in_attr = attrs;
 	ios->in_attr_len = ARRAY_SIZE(attrs);
 
@@ -339,7 +321,7 @@ int exofs_sbi_write_stats(struct exofs_sb_info *sbi)
 	struct exofs_io_state *ios;
 	int ret;
 
-	ret = exofs_get_io_state(&sbi->layout, &ios);
+	ret = exofs_get_io_state(&sbi->layout, &sbi->comps, &ios);
 	if (unlikely(ret)) {
 		EXOFS_ERR("%s: exofs_get_io_state failed.\n", __func__);
 		return ret;
@@ -349,7 +331,7 @@ int exofs_sbi_write_stats(struct exofs_sb_info *sbi)
 	sbi->s_ess.s_numfiles = cpu_to_le64(sbi->s_numfiles);
 	attrs[0].val_ptr = &sbi->s_ess;
 
-	ios->cred = sbi->s_cred;
+
 	ios->done = stats_done;
 	ios->private = sbi;
 	ios->out_attr = attrs;
@@ -377,6 +359,8 @@ int exofs_sync_fs(struct super_block *sb, int wait)
 {
 	struct exofs_sb_info *sbi;
 	struct exofs_fscb *fscb;
+	struct exofs_comp one_comp;
+	struct exofs_components comps;
 	struct exofs_io_state *ios;
 	int ret = -ENOMEM;
 
@@ -393,7 +377,10 @@ int exofs_sync_fs(struct super_block *sb, int wait)
 	 * version). Otherwise the exofs_fscb is read-only from mkfs time. All
 	 * the writeable info is set in exofs_sbi_write_stats() above.
 	 */
-	ret = exofs_get_io_state(&sbi->layout, &ios);
+
+	exofs_init_comps(&comps, &one_comp, sbi, EXOFS_SUPER_ID);
+
+	ret = exofs_get_io_state(&sbi->layout, &comps, &ios);
 	if (unlikely(ret))
 		goto out;
 
@@ -407,10 +394,8 @@ int exofs_sync_fs(struct super_block *sb, int wait)
 	fscb->s_newfs = 0;
 	fscb->s_version = EXOFS_FSCB_VER;
 
-	ios->obj.id = EXOFS_SUPER_ID;
 	ios->offset = 0;
 	ios->kern_buff = fscb;
-	ios->cred = sbi->s_cred;
 
 	ret = exofs_sbi_write(ios);
 	if (unlikely(ret))
@@ -446,17 +431,17 @@ static void _exofs_print_device(const char *msg, const char *dev_path,
 
 void exofs_free_sbi(struct exofs_sb_info *sbi)
 {
-	while (sbi->layout.s_numdevs) {
-		int i = --sbi->layout.s_numdevs;
-		struct osd_dev *od = sbi->layout.s_ods[i];
+	while (sbi->comps.numdevs) {
+		int i = --sbi->comps.numdevs;
+		struct osd_dev *od = sbi->comps.ods[i];
 
 		if (od) {
-			sbi->layout.s_ods[i] = NULL;
+			sbi->comps.ods[i] = NULL;
 			osduld_put_device(od);
 		}
 	}
-	if (sbi->layout.s_ods != sbi->_min_one_dev)
-		kfree(sbi->layout.s_ods);
+	if (sbi->comps.ods != sbi->_min_one_dev)
+		kfree(sbi->comps.ods);
 	kfree(sbi);
 }
 
@@ -483,8 +468,8 @@ static void exofs_put_super(struct super_block *sb)
 				  msecs_to_jiffies(100));
 	}
 
-	_exofs_print_device("Unmounting", NULL, sbi->layout.s_ods[0],
-			    sbi->layout.s_pid);
+	_exofs_print_device("Unmounting", NULL, sbi->comps.ods[0],
+			    sbi->one_comp.obj.partition);
 
 	bdi_destroy(&sbi->bdi);
 	exofs_free_sbi(sbi);
@@ -624,8 +609,7 @@ static int exofs_read_lookup_dev_table(struct exofs_sb_info *sbi,
 				       struct osd_dev *fscb_od,
 				       unsigned table_count)
 {
-	struct osd_obj_id obj = {.partition = sbi->layout.s_pid,
-				 .id = EXOFS_DEVTABLE_ID};
+	struct exofs_comp comp;
 	struct exofs_device_table *dt;
 	unsigned table_bytes = table_count * sizeof(dt->dt_dev_table[0]) +
 					     sizeof(*dt);
@@ -639,8 +623,14 @@ static int exofs_read_lookup_dev_table(struct exofs_sb_info *sbi,
 		return -ENOMEM;
 	}
 
-	sbi->layout.s_numdevs = 0;
-	ret = exofs_read_kern(fscb_od, sbi->s_cred, &obj, 0, dt, table_bytes);
+	sbi->comps.numdevs = 0;
+
+	comp.obj.partition = sbi->one_comp.obj.partition;
+	comp.obj.id = EXOFS_DEVTABLE_ID;
+	exofs_make_credential(comp.cred, &comp.obj);
+
+	ret = exofs_read_kern(fscb_od, comp.cred, &comp.obj, 0, dt,
+			      table_bytes);
 	if (unlikely(ret)) {
 		EXOFS_ERR("ERROR: reading device table\n");
 		goto out;
@@ -658,10 +648,13 @@ static int exofs_read_lookup_dev_table(struct exofs_sb_info *sbi,
 		goto out;
 
 	if (likely(numdevs > 1)) {
-		unsigned size = numdevs * sizeof(sbi->layout.s_ods[0]);
+		unsigned size = numdevs * sizeof(sbi->comps.ods[0]);
 
-		sbi->layout.s_ods = kzalloc(size, GFP_KERNEL);
-		if (unlikely(!sbi->layout.s_ods)) {
+		/* Twice bigger table: See exofs_init_comps() and below
+		 * comment
+		 */
+		sbi->comps.ods = kzalloc(size + size - 1, GFP_KERNEL);
+		if (unlikely(!sbi->comps.ods)) {
 			EXOFS_ERR("ERROR: faild allocating Device array[%d]\n",
 				  numdevs);
 			ret = -ENOMEM;
@@ -688,8 +681,8 @@ static int exofs_read_lookup_dev_table(struct exofs_sb_info *sbi,
 		 * line. We always keep them in device-table order.
 		 */
 		if (fscb_od && osduld_device_same(fscb_od, &odi)) {
-			sbi->layout.s_ods[i] = fscb_od;
-			++sbi->layout.s_numdevs;
+			sbi->comps.ods[i] = fscb_od;
+			++sbi->comps.numdevs;
 			fscb_od = NULL;
 			continue;
 		}
@@ -702,13 +695,13 @@ static int exofs_read_lookup_dev_table(struct exofs_sb_info *sbi,
 			goto out;
 		}
 
-		sbi->layout.s_ods[i] = od;
-		++sbi->layout.s_numdevs;
+		sbi->comps.ods[i] = od;
+		++sbi->comps.numdevs;
 
 		/* Read the fscb of the other devices to make sure the FS
 		 * partition is there.
 		 */
-		ret = exofs_read_kern(od, sbi->s_cred, &obj, 0, &fscb,
+		ret = exofs_read_kern(od, comp.cred, &comp.obj, 0, &fscb,
 				      sizeof(fscb));
 		if (unlikely(ret)) {
 			EXOFS_ERR("ERROR: Malformed participating device "
@@ -725,13 +718,22 @@ static int exofs_read_lookup_dev_table(struct exofs_sb_info *sbi,
 
 out:
 	kfree(dt);
-	if (unlikely(!ret && fscb_od)) {
-		EXOFS_ERR(
-		      "ERROR: Bad device-table container device not present\n");
+	if (likely(!ret)) {
+		unsigned numdevs = sbi->comps.numdevs;
+
+		if (unlikely(fscb_od)) {
+			EXOFS_ERR("ERROR: Bad device-table container device not present\n");
 			osduld_put_device(fscb_od);
-		ret = -EINVAL;
+			return -EINVAL;
+		}
+		/* exofs round-robins the device table view according to inode
+		 * number. We hold a: twice bigger table hence inodes can point
+		 * to any device and have a sequential view of the table
+		 * starting at this device. See exofs_init_comps()
+		 */
+		for (i = 0; i < numdevs - 1; ++i)
+			sbi->comps.ods[i + numdevs] = sbi->comps.ods[i];
 	}
-
 	return ret;
 }
 
@@ -745,7 +747,7 @@ static int exofs_fill_super(struct super_block *sb, void *data, int silent)
 	struct exofs_sb_info *sbi;	/*extended info                  */
 	struct osd_dev *od;		/* Master device                 */
 	struct exofs_fscb fscb;		/*on-disk superblock info        */
-	struct osd_obj_id obj;
+	struct exofs_comp comp;
 	unsigned table_count;
 	int ret;
 
@@ -776,11 +778,16 @@ static int exofs_fill_super(struct super_block *sb, void *data, int silent)
 	sbi->layout.group_width = 1;
 	sbi->layout.group_depth = -1;
 	sbi->layout.group_count = 1;
-	sbi->layout.s_ods = sbi->_min_one_dev;
-	sbi->layout.s_numdevs = 1;
-	sbi->layout.s_pid = opts->pid;
 	sbi->s_timeout = opts->timeout;
 
+	sbi->one_comp.obj.partition = opts->pid;
+	sbi->one_comp.obj.id = 0;
+	exofs_make_credential(sbi->one_comp.cred, &sbi->one_comp.obj);
+	sbi->comps.numdevs = 1;
+	sbi->comps.single_comp = EC_SINGLE_COMP;
+	sbi->comps.comps = &sbi->one_comp;
+	sbi->comps.ods = sbi->_min_one_dev;
+
 	/* fill in some other data by hand */
 	memset(sb->s_id, 0, sizeof(sb->s_id));
 	strcpy(sb->s_id, "exofs");
@@ -791,11 +798,11 @@ static int exofs_fill_super(struct super_block *sb, void *data, int silent)
 	sb->s_bdev = NULL;
 	sb->s_dev = 0;
 
-	obj.partition = sbi->layout.s_pid;
-	obj.id = EXOFS_SUPER_ID;
-	exofs_make_credential(sbi->s_cred, &obj);
+	comp.obj.partition = sbi->one_comp.obj.partition;
+	comp.obj.id = EXOFS_SUPER_ID;
+	exofs_make_credential(comp.cred, &comp.obj);
 
-	ret = exofs_read_kern(od, sbi->s_cred, &obj, 0, &fscb, sizeof(fscb));
+	ret = exofs_read_kern(od, comp.cred, &comp.obj, 0, &fscb, sizeof(fscb));
 	if (unlikely(ret))
 		goto free_sbi;
 
@@ -828,7 +835,7 @@ static int exofs_fill_super(struct super_block *sb, void *data, int silent)
 		if (unlikely(ret))
 			goto free_sbi;
 	} else {
-		sbi->layout.s_ods[0] = od;
+		sbi->comps.ods[0] = od;
 	}
 
 	__sbi_read_stats(sbi);
@@ -868,13 +875,13 @@ static int exofs_fill_super(struct super_block *sb, void *data, int silent)
 		goto free_sbi;
 	}
 
-	_exofs_print_device("Mounting", opts->dev_name, sbi->layout.s_ods[0],
-			    sbi->layout.s_pid);
+	_exofs_print_device("Mounting", opts->dev_name, sbi->comps.ods[0],
+			    sbi->one_comp.obj.partition);
 	return 0;
 
 free_sbi:
 	EXOFS_ERR("Unable to mount exofs on %s pid=0x%llx err=%d\n",
-		  opts->dev_name, sbi->layout.s_pid, ret);
+		  opts->dev_name, sbi->one_comp.obj.partition, ret);
 	exofs_free_sbi(sbi);
 	return ret;
 }
@@ -915,17 +922,14 @@ static int exofs_statfs(struct dentry *dentry, struct kstatfs *buf)
 	};
 	uint64_t capacity = ULLONG_MAX;
 	uint64_t used = ULLONG_MAX;
-	uint8_t cred_a[OSD_CAP_LEN];
 	int ret;
 
-	ret = exofs_get_io_state(&sbi->layout, &ios);
+	ret = exofs_get_io_state(&sbi->layout, &sbi->comps, &ios);
 	if (ret) {
 		EXOFS_DBGMSG("exofs_get_io_state failed.\n");
 		return ret;
 	}
 
-	exofs_make_credential(cred_a, &ios->obj);
-	ios->cred = sbi->s_cred;
 	ios->in_attr = attrs;
 	ios->in_attr_len = ARRAY_SIZE(attrs);