loop.c 29 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4
/*
 *  linux/drivers/block/loop.c
 *
 *  Written by Theodore Ts'o, 3/29/93
Andries E. Brouwer's avatar
Andries E. Brouwer committed
5
 *
Linus Torvalds's avatar
Linus Torvalds committed
6
 * Copyright 1993 by Theodore Ts'o.  Redistribution of this file is
Linus Torvalds's avatar
Linus Torvalds committed
7
 * permitted under the GNU General Public License.
Linus Torvalds's avatar
Linus Torvalds committed
8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 *
 * DES encryption plus some minor changes by Werner Almesberger, 30-MAY-1993
 * more DES encryption plus IDEA encryption by Nicholas J. Leon, June 20, 1996
 *
 * Modularized and updated for 1.1.16 kernel - Mitch Dsouza 28th May 1994
 * Adapted for 1.3.59 kernel - Andries Brouwer, 1 Feb 1996
 *
 * Fixed do_loop_request() re-entrancy - Vincent.Renardias@waw.com Mar 20, 1997
 *
 * Added devfs support - Richard Gooch <rgooch@atnf.csiro.au> 16-Jan-1998
 *
 * Handle sparse backing files correctly - Kenn Humborg, Jun 28, 1998
 *
 * Loadable modules and other fixes by AK, 1998
 *
 * Make real block number available to downstream transfer functions, enables
Andries E. Brouwer's avatar
Andries E. Brouwer committed
24
 * CBC (and relatives) mode encryption requiring unique IVs per data block.
Linus Torvalds's avatar
Linus Torvalds committed
25 26 27 28
 * Reed H. Petty, rhp@draper.net
 *
 * Maximum number of loop devices now dynamic via max_loop module parameter.
 * Russell Kroll <rkroll@exploits.org> 19990701
Andries E. Brouwer's avatar
Andries E. Brouwer committed
29
 *
Linus Torvalds's avatar
Linus Torvalds committed
30 31 32 33
 * Maximum number of loop devices when compiled-in now selectable by passing
 * max_loop=<1-255> to the kernel on boot.
 * Erik I. Bols, <eriki@himolde.no>, Oct 31, 1999
 *
Linus Torvalds's avatar
Linus Torvalds committed
34 35 36 37 38
 * Completely rewrite request handling to be make_request_fn style and
 * non blocking, pushing work to a helper thread. Lots of fixes from
 * Al Viro too.
 * Jens Axboe <axboe@suse.de>, Nov 2000
 *
39 40 41
 * Support up to 256 loop devices
 * Heinz Mauelshagen <mge@sistina.com>, Feb 2002
 *
Linus Torvalds's avatar
Linus Torvalds committed
42
 * Still To Fix:
Andries E. Brouwer's avatar
Andries E. Brouwer committed
43 44
 * - Advisory locking is ignored here.
 * - Should use an own CAP_* category instead of CAP_SYS_ADMIN
Linus Torvalds's avatar
Linus Torvalds committed
45
 *
Andries E. Brouwer's avatar
Andries E. Brouwer committed
46
 */
Linus Torvalds's avatar
Linus Torvalds committed
47

Linus Torvalds's avatar
Linus Torvalds committed
48
#include <linux/config.h>
Linus Torvalds's avatar
Linus Torvalds committed
49 50 51 52 53 54 55 56
#include <linux/module.h>

#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/stat.h>
#include <linux/errno.h>
#include <linux/major.h>
Linus Torvalds's avatar
Linus Torvalds committed
57
#include <linux/wait.h>
Linus Torvalds's avatar
Linus Torvalds committed
58
#include <linux/blkpg.h>
Linus Torvalds's avatar
Linus Torvalds committed
59 60
#include <linux/init.h>
#include <linux/devfs_fs_kernel.h>
Linus Torvalds's avatar
Linus Torvalds committed
61 62 63
#include <linux/smp_lock.h>
#include <linux/swap.h>
#include <linux/slab.h>
Pavel Machek's avatar
Pavel Machek committed
64 65
#include <linux/loop.h>
#include <linux/suspend.h>
Andrew Morton's avatar
Andrew Morton committed
66
#include <linux/writeback.h>
67
#include <linux/buffer_head.h>		/* for invalidate_bdev() */
Linus Torvalds's avatar
Linus Torvalds committed
68 69 70 71 72

#include <asm/uaccess.h>

static int max_loop = 8;
static struct loop_device *loop_dev;
73
static struct gendisk **disks;
Linus Torvalds's avatar
Linus Torvalds committed
74 75 76 77 78

/*
 * Transfer functions
 */
static int transfer_none(struct loop_device *lo, int cmd, char *raw_buf,
79
			 char *loop_buf, int size, sector_t real_block)
Linus Torvalds's avatar
Linus Torvalds committed
80
{
Linus Torvalds's avatar
Linus Torvalds committed
81 82 83 84 85 86
	if (raw_buf != loop_buf) {
		if (cmd == READ)
			memcpy(loop_buf, raw_buf, size);
		else
			memcpy(raw_buf, loop_buf, size);
	}
Linus Torvalds's avatar
Linus Torvalds committed
87

Linus Torvalds's avatar
Linus Torvalds committed
88 89 90 91
	return 0;
}

static int transfer_xor(struct loop_device *lo, int cmd, char *raw_buf,
92
			char *loop_buf, int size, sector_t real_block)
Linus Torvalds's avatar
Linus Torvalds committed
93 94 95 96 97 98 99 100 101 102 103
{
	char	*in, *out, *key;
	int	i, keysize;

	if (cmd == READ) {
		in = raw_buf;
		out = loop_buf;
	} else {
		in = loop_buf;
		out = raw_buf;
	}
Linus Torvalds's avatar
Linus Torvalds committed
104

Linus Torvalds's avatar
Linus Torvalds committed
105 106
	key = lo->lo_encrypt_key;
	keysize = lo->lo_encrypt_key_size;
Linus Torvalds's avatar
Linus Torvalds committed
107
	for (i = 0; i < size; i++)
Linus Torvalds's avatar
Linus Torvalds committed
108 109 110 111
		*out++ = *in++ ^ key[(i & 511) % keysize];
	return 0;
}

112
static int xor_init(struct loop_device *lo, const struct loop_info64 *info)
Linus Torvalds's avatar
Linus Torvalds committed
113 114 115 116 117 118
{
	if (info->lo_encrypt_key_size <= 0)
		return -EINVAL;
	return 0;
}

119
static struct loop_func_table none_funcs = {
120 121
	.number = LO_CRYPT_NONE,
	.transfer = transfer_none,
Linus Torvalds's avatar
Linus Torvalds committed
122 123
}; 	

124
static struct loop_func_table xor_funcs = {
125 126
	.number = LO_CRYPT_XOR,
	.transfer = transfer_xor,
127
	.init = xor_init
Linus Torvalds's avatar
Linus Torvalds committed
128 129
}; 	

Andries E. Brouwer's avatar
Andries E. Brouwer committed
130
/* xfer_funcs[0] is special - its release function is never called */
131
static struct loop_func_table *xfer_funcs[MAX_LO_CRYPT] = {
Linus Torvalds's avatar
Linus Torvalds committed
132
	&none_funcs,
Andries E. Brouwer's avatar
Andries E. Brouwer committed
133
	&xor_funcs
Linus Torvalds's avatar
Linus Torvalds committed
134 135
};

Andries E. Brouwer's avatar
Andries E. Brouwer committed
136 137
static int
figure_loop_size(struct loop_device *lo)
Linus Torvalds's avatar
Linus Torvalds committed
138
{
139
	loff_t size, offset, loopsize;
140
	sector_t x;
141 142

	/* Compute loopsize in bytes */
143
	size = i_size_read(lo->lo_backing_file->f_mapping->host);
144 145 146 147 148
	offset = lo->lo_offset;
	loopsize = size - offset;
	if (lo->lo_sizelimit > 0 && lo->lo_sizelimit < loopsize)
		loopsize = lo->lo_sizelimit;

149 150 151 152
	/*
	 * Unfortunately, if we want to do I/O on the device,
	 * the number of 512-byte sectors has to fit into a sector_t.
	 */
153
	size = loopsize >> 9;
154
	x = (sector_t)size;
Andries E. Brouwer's avatar
Andries E. Brouwer committed
155

156 157 158
	if ((loff_t)x != size)
		return -EFBIG;

Andries E. Brouwer's avatar
Andries E. Brouwer committed
159
	set_capacity(disks[lo->lo_number], x);
160
	return 0;					
Linus Torvalds's avatar
Linus Torvalds committed
161 162
}

Andries E. Brouwer's avatar
Andries E. Brouwer committed
163 164 165
static inline int
lo_do_transfer(struct loop_device *lo, int cmd, char *rbuf,
	       char *lbuf, int size, sector_t rblock)
166 167 168 169 170 171 172
{
	if (!lo->transfer)
		return 0;

	return lo->transfer(lo, cmd, rbuf, lbuf, size, rblock);
}

173 174
static int
do_lo_send(struct loop_device *lo, struct bio_vec *bvec, int bsize, loff_t pos)
Linus Torvalds's avatar
Linus Torvalds committed
175 176
{
	struct file *file = lo->lo_backing_file; /* kudos to NFsckingS */
177
	struct address_space *mapping = file->f_mapping;
Linus Torvalds's avatar
Linus Torvalds committed
178 179
	struct address_space_operations *aops = mapping->a_ops;
	struct page *page;
Linus Torvalds's avatar
Linus Torvalds committed
180
	char *kaddr, *data;
181
	pgoff_t index;
Linus Torvalds's avatar
Linus Torvalds committed
182
	unsigned size, offset;
Linus Torvalds's avatar
Linus Torvalds committed
183
	int len;
184
	int ret = 0;
Linus Torvalds's avatar
Linus Torvalds committed
185

Linus Torvalds's avatar
Linus Torvalds committed
186
	down(&mapping->host->i_sem);
Linus Torvalds's avatar
Linus Torvalds committed
187
	index = pos >> PAGE_CACHE_SHIFT;
188
	offset = pos & ((pgoff_t)PAGE_CACHE_SIZE - 1);
189 190
	data = kmap(bvec->bv_page) + bvec->bv_offset;
	len = bvec->bv_len;
Linus Torvalds's avatar
Linus Torvalds committed
191
	while (len > 0) {
192
		sector_t IV;
Linus Torvalds's avatar
Linus Torvalds committed
193 194
		int transfer_result;

195 196
		IV = ((sector_t)index << (PAGE_CACHE_SHIFT - 9))+(offset >> 9);

Linus Torvalds's avatar
Linus Torvalds committed
197 198 199 200 201 202 203 204 205
		size = PAGE_CACHE_SIZE - offset;
		if (size > len)
			size = len;

		page = grab_cache_page(mapping, index);
		if (!page)
			goto fail;
		if (aops->prepare_write(file, page, offset, offset+size))
			goto unlock;
206
		kaddr = kmap(page);
207 208
		transfer_result = lo_do_transfer(lo, WRITE, kaddr + offset,
						 data, size, IV);
Linus Torvalds's avatar
Linus Torvalds committed
209 210 211 212 213
		if (transfer_result) {
			/*
			 * The transfer failed, but we still write the data to
			 * keep prepare/commit calls balanced.
			 */
214 215
			printk(KERN_ERR "loop: transfer error block %llu\n",
			       (unsigned long long)index);
Linus Torvalds's avatar
Linus Torvalds committed
216 217
			memset(kaddr + offset, 0, size);
		}
218 219
		flush_dcache_page(page);
		kunmap(page);
Linus Torvalds's avatar
Linus Torvalds committed
220 221
		if (aops->commit_write(file, page, offset, offset+size))
			goto unlock;
Linus Torvalds's avatar
Linus Torvalds committed
222 223
		if (transfer_result)
			goto unlock;
Linus Torvalds's avatar
Linus Torvalds committed
224 225 226 227 228
		data += size;
		len -= size;
		offset = 0;
		index++;
		pos += size;
Andrew Morton's avatar
Andrew Morton committed
229
		unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
230 231
		page_cache_release(page);
	}
Linus Torvalds's avatar
Linus Torvalds committed
232
	up(&mapping->host->i_sem);
233 234 235
out:
	kunmap(bvec->bv_page);
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
236 237

unlock:
Andrew Morton's avatar
Andrew Morton committed
238
	unlock_page(page);
Linus Torvalds's avatar
Linus Torvalds committed
239 240
	page_cache_release(page);
fail:
Linus Torvalds's avatar
Linus Torvalds committed
241
	up(&mapping->host->i_sem);
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260
	ret = -1;
	goto out;
}

static int
lo_send(struct loop_device *lo, struct bio *bio, int bsize, loff_t pos)
{
	unsigned vecnr;
	int ret = 0;

	for (vecnr = 0; vecnr < bio->bi_vcnt; vecnr++) {
		struct bio_vec *bvec = &bio->bi_io_vec[vecnr];

		ret = do_lo_send(lo, bvec, bsize, pos);
		if (ret < 0)
			break;
		pos += bvec->bv_len;
	}
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
261 262 263 264 265
}

struct lo_read_data {
	struct loop_device *lo;
	char *data;
Linus Torvalds's avatar
Linus Torvalds committed
266
	int bsize;
Linus Torvalds's avatar
Linus Torvalds committed
267 268
};

269 270 271
static int
lo_read_actor(read_descriptor_t *desc, struct page *page,
	      unsigned long offset, unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
272 273 274 275 276
{
	char *kaddr;
	unsigned long count = desc->count;
	struct lo_read_data *p = (struct lo_read_data*)desc->buf;
	struct loop_device *lo = p->lo;
277 278 279
	sector_t IV;

	IV = ((sector_t) page->index << (PAGE_CACHE_SHIFT - 9))+(offset >> 9);
Linus Torvalds's avatar
Linus Torvalds committed
280 281 282 283 284

	if (size > count)
		size = count;

	kaddr = kmap(page);
Linus Torvalds's avatar
Linus Torvalds committed
285
	if (lo_do_transfer(lo, READ, kaddr + offset, p->data, size, IV)) {
Linus Torvalds's avatar
Linus Torvalds committed
286
		size = 0;
287 288
		printk(KERN_ERR "loop: transfer error block %ld\n",
		       page->index);
Linus Torvalds's avatar
Linus Torvalds committed
289 290 291 292 293 294 295 296 297 298
		desc->error = -EINVAL;
	}
	kunmap(page);
	
	desc->count = count - size;
	desc->written += size;
	p->data += size;
	return size;
}

299 300
static int
do_lo_receive(struct loop_device *lo,
301
	      struct bio_vec *bvec, int bsize, loff_t pos)
Linus Torvalds's avatar
Linus Torvalds committed
302 303
{
	struct lo_read_data cookie;
Linus Torvalds's avatar
Linus Torvalds committed
304
	struct file *file;
Hugh Dickins's avatar
Hugh Dickins committed
305
	int retval;
Linus Torvalds's avatar
Linus Torvalds committed
306 307

	cookie.lo = lo;
308
	cookie.data = kmap(bvec->bv_page) + bvec->bv_offset;
Linus Torvalds's avatar
Linus Torvalds committed
309
	cookie.bsize = bsize;
Linus Torvalds's avatar
Linus Torvalds committed
310
	file = lo->lo_backing_file;
Hugh Dickins's avatar
Hugh Dickins committed
311
	retval = file->f_op->sendfile(file, &pos, bvec->bv_len,
312
			lo_read_actor, &cookie);
313
	kunmap(bvec->bv_page);
Hugh Dickins's avatar
Hugh Dickins committed
314
	return (retval < 0)? retval: 0;
Linus Torvalds's avatar
Linus Torvalds committed
315
}
316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332

static int
lo_receive(struct loop_device *lo, struct bio *bio, int bsize, loff_t pos)
{
	unsigned vecnr;
	int ret = 0;

	for (vecnr = 0; vecnr < bio->bi_vcnt; vecnr++) {
		struct bio_vec *bvec = &bio->bi_io_vec[vecnr];

		ret = do_lo_receive(lo, bvec, bsize, pos);
		if (ret < 0)
			break;
		pos += bvec->bv_len;
	}
	return ret;
}
Linus Torvalds's avatar
Linus Torvalds committed
333

Linus Torvalds's avatar
Linus Torvalds committed
334
static int do_bio_filebacked(struct loop_device *lo, struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
335 336
{
	loff_t pos;
Linus Torvalds's avatar
Linus Torvalds committed
337 338
	int ret;

Linus Torvalds's avatar
Linus Torvalds committed
339
	pos = ((loff_t) bio->bi_sector << 9) + lo->lo_offset;
340 341 342 343
	if (bio_rw(bio) == WRITE)
		ret = lo_send(lo, bio, lo->lo_blocksize, pos);
	else
		ret = lo_receive(lo, bio, lo->lo_blocksize, pos);
Linus Torvalds's avatar
Linus Torvalds committed
344 345 346
	return ret;
}

347
static int loop_end_io_transfer(struct bio *, unsigned int, int);
348

Linus Torvalds's avatar
Linus Torvalds committed
349
static void loop_put_buffer(struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
350
{
Linus Torvalds's avatar
Linus Torvalds committed
351
	/*
Linus Torvalds's avatar
Linus Torvalds committed
352
	 * check bi_end_io, may just be a remapped bio
Linus Torvalds's avatar
Linus Torvalds committed
353
	 */
Linus Torvalds's avatar
Linus Torvalds committed
354
	if (bio && bio->bi_end_io == loop_end_io_transfer) {
Linus Torvalds's avatar
Linus Torvalds committed
355
		int i;
356

Linus Torvalds's avatar
Linus Torvalds committed
357 358 359
		for (i = 0; i < bio->bi_vcnt; i++)
			__free_page(bio->bi_io_vec[i].bv_page);

Linus Torvalds's avatar
Linus Torvalds committed
360
		bio_put(bio);
Linus Torvalds's avatar
Linus Torvalds committed
361 362 363 364
	}
}

/*
Linus Torvalds's avatar
Linus Torvalds committed
365
 * Add bio to back of pending list
Linus Torvalds's avatar
Linus Torvalds committed
366
 */
Linus Torvalds's avatar
Linus Torvalds committed
367
static void loop_add_bio(struct loop_device *lo, struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
368 369 370 371
{
	unsigned long flags;

	spin_lock_irqsave(&lo->lo_lock, flags);
Linus Torvalds's avatar
Linus Torvalds committed
372 373 374
	if (lo->lo_biotail) {
		lo->lo_biotail->bi_next = bio;
		lo->lo_biotail = bio;
Linus Torvalds's avatar
Linus Torvalds committed
375
	} else
Linus Torvalds's avatar
Linus Torvalds committed
376
		lo->lo_bio = lo->lo_biotail = bio;
Linus Torvalds's avatar
Linus Torvalds committed
377 378 379 380 381 382 383 384
	spin_unlock_irqrestore(&lo->lo_lock, flags);

	up(&lo->lo_bh_mutex);
}

/*
 * Grab first pending buffer
 */
Linus Torvalds's avatar
Linus Torvalds committed
385
static struct bio *loop_get_bio(struct loop_device *lo)
Linus Torvalds's avatar
Linus Torvalds committed
386
{
Linus Torvalds's avatar
Linus Torvalds committed
387
	struct bio *bio;
Linus Torvalds's avatar
Linus Torvalds committed
388

Linus Torvalds's avatar
Linus Torvalds committed
389
	spin_lock_irq(&lo->lo_lock);
Linus Torvalds's avatar
Linus Torvalds committed
390 391 392 393 394
	if ((bio = lo->lo_bio)) {
		if (bio == lo->lo_biotail)
			lo->lo_biotail = NULL;
		lo->lo_bio = bio->bi_next;
		bio->bi_next = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
395 396 397
	}
	spin_unlock_irq(&lo->lo_lock);

Linus Torvalds's avatar
Linus Torvalds committed
398
	return bio;
Linus Torvalds's avatar
Linus Torvalds committed
399 400 401
}

/*
Linus Torvalds's avatar
Linus Torvalds committed
402 403 404
 * if this was a WRITE lo->transfer stuff has already been done. for READs,
 * queue it for the loop thread and let it do the transfer out of
 * bi_end_io context (we don't want to do decrypt of a page with irqs
Linus Torvalds's avatar
Linus Torvalds committed
405 406
 * disabled)
 */
407
static int loop_end_io_transfer(struct bio *bio, unsigned int bytes_done, int err)
Linus Torvalds's avatar
Linus Torvalds committed
408
{
Linus Torvalds's avatar
Linus Torvalds committed
409
	struct bio *rbh = bio->bi_private;
Alexander Viro's avatar
Alexander Viro committed
410
	struct loop_device *lo = rbh->bi_bdev->bd_disk->private_data;
Linus Torvalds's avatar
Linus Torvalds committed
411

412 413 414
	if (bio->bi_size)
		return 1;

Alexander Viro's avatar
Alexander Viro committed
415
	if (err || bio_rw(bio) == WRITE) {
416
		bio_endio(rbh, rbh->bi_size, err);
Linus Torvalds's avatar
Linus Torvalds committed
417 418
		if (atomic_dec_and_test(&lo->lo_pending))
			up(&lo->lo_bh_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
419
		loop_put_buffer(bio);
Linus Torvalds's avatar
Linus Torvalds committed
420
	} else
Linus Torvalds's avatar
Linus Torvalds committed
421
		loop_add_bio(lo, bio);
422 423

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
424 425
}

426
static struct bio *loop_copy_bio(struct bio *rbh)
427
{
428
	struct bio *bio;
429 430 431
	struct bio_vec *bv;
	int i;

432
	bio = bio_alloc(__GFP_NOWARN, rbh->bi_vcnt);
433
	if (!bio)
434 435 436
		return NULL;

	/*
437
	 * iterate iovec list and alloc pages
438
	 */
439 440
	__bio_for_each_segment(bv, rbh, i, 0) {
		struct bio_vec *bbv = &bio->bi_io_vec[i];
441

442
		bbv->bv_page = alloc_page(__GFP_NOWARN|__GFP_HIGHMEM);
443 444 445 446 447 448 449
		if (bbv->bv_page == NULL)
			goto oom;

		bbv->bv_len = bv->bv_len;
		bbv->bv_offset = bv->bv_offset;
	}

450 451
	bio->bi_vcnt = rbh->bi_vcnt;
	bio->bi_size = rbh->bi_size;
452

453
	return bio;
454 455 456

oom:
	while (--i >= 0)
457
		__free_page(bio->bi_io_vec[i].bv_page);
458 459 460 461 462

	bio_put(bio);
	return NULL;
}

Linus Torvalds's avatar
Linus Torvalds committed
463
static struct bio *loop_get_buffer(struct loop_device *lo, struct bio *rbh)
Linus Torvalds's avatar
Linus Torvalds committed
464
{
Linus Torvalds's avatar
Linus Torvalds committed
465
	struct bio *bio;
Linus Torvalds's avatar
Linus Torvalds committed
466

467 468 469 470 471 472 473 474 475 476
	/*
	 * When called on the page reclaim -> writepage path, this code can
	 * trivially consume all memory.  So we drop PF_MEMALLOC to avoid
	 * stealing all the page reserves and throttle to the writeout rate.
	 * pdflush will have been woken by page reclaim.  Let it do its work.
	 */
	do {
		int flags = current->flags;

		current->flags &= ~PF_MEMALLOC;
477
		bio = loop_copy_bio(rbh);
478 479 480
		if (flags & PF_MEMALLOC)
			current->flags |= PF_MEMALLOC;

481 482 483
		if (bio == NULL)
			blk_congestion_wait(WRITE, HZ/10);
	} while (bio == NULL);
Linus Torvalds's avatar
Linus Torvalds committed
484 485 486 487 488

	bio->bi_end_io = loop_end_io_transfer;
	bio->bi_private = rbh;
	bio->bi_sector = rbh->bi_sector + (lo->lo_offset >> 9);
	bio->bi_rw = rbh->bi_rw;
489
	bio->bi_bdev = lo->lo_device;
Linus Torvalds's avatar
Linus Torvalds committed
490

Linus Torvalds's avatar
Linus Torvalds committed
491
	return bio;
Linus Torvalds's avatar
Linus Torvalds committed
492 493
}

494 495
static int loop_transfer_bio(struct loop_device *lo,
			     struct bio *to_bio, struct bio *from_bio)
Linus Torvalds's avatar
Linus Torvalds committed
496
{
497
	sector_t IV;
498 499 500 501
	struct bio_vec *from_bvec, *to_bvec;
	char *vto, *vfrom;
	int ret = 0, i;

502 503
	IV = from_bio->bi_sector + (lo->lo_offset >> 9);

504 505 506 507 508 509 510 511 512 513 514
	__bio_for_each_segment(from_bvec, from_bio, i, 0) {
		to_bvec = &to_bio->bi_io_vec[i];

		kmap(from_bvec->bv_page);
		kmap(to_bvec->bv_page);
		vfrom = page_address(from_bvec->bv_page) + from_bvec->bv_offset;
		vto = page_address(to_bvec->bv_page) + to_bvec->bv_offset;
		ret |= lo_do_transfer(lo, bio_data_dir(to_bio), vto, vfrom,
					from_bvec->bv_len, IV);
		kunmap(from_bvec->bv_page);
		kunmap(to_bvec->bv_page);
515
		IV += from_bvec->bv_len >> 9;
516 517 518 519 520 521 522 523
	}

	return ret;
}
		
static int loop_make_request(request_queue_t *q, struct bio *old_bio)
{
	struct bio *new_bio = NULL;
524
	struct loop_device *lo = q->queuedata;
525
	int rw = bio_rw(old_bio);
Linus Torvalds's avatar
Linus Torvalds committed
526

527
	if (!lo)
Linus Torvalds's avatar
Linus Torvalds committed
528 529 530 531 532
		goto out;

	spin_lock_irq(&lo->lo_lock);
	if (lo->lo_state != Lo_bound)
		goto inactive;
Linus Torvalds's avatar
Linus Torvalds committed
533
	atomic_inc(&lo->lo_pending);
Linus Torvalds's avatar
Linus Torvalds committed
534 535 536
	spin_unlock_irq(&lo->lo_lock);

	if (rw == WRITE) {
Linus Torvalds's avatar
Linus Torvalds committed
537
		if (lo->lo_flags & LO_FLAGS_READ_ONLY)
Linus Torvalds's avatar
Linus Torvalds committed
538 539 540 541
			goto err;
	} else if (rw == READA) {
		rw = READ;
	} else if (rw != READ) {
Linus Torvalds's avatar
Linus Torvalds committed
542
		printk(KERN_ERR "loop: unknown command (%x)\n", rw);
Linus Torvalds's avatar
Linus Torvalds committed
543
		goto err;
Linus Torvalds's avatar
Linus Torvalds committed
544 545
	}

Linus Torvalds's avatar
Linus Torvalds committed
546 547 548 549
	/*
	 * file backed, queue for loop_thread to handle
	 */
	if (lo->lo_flags & LO_FLAGS_DO_BMAP) {
550
		loop_add_bio(lo, old_bio);
Linus Torvalds's avatar
Linus Torvalds committed
551 552
		return 0;
	}
Linus Torvalds's avatar
Linus Torvalds committed
553

Linus Torvalds's avatar
Linus Torvalds committed
554 555 556
	/*
	 * piggy old buffer on original, and submit for I/O
	 */
557
	new_bio = loop_get_buffer(lo, old_bio);
Linus Torvalds's avatar
Linus Torvalds committed
558
	if (rw == WRITE) {
559
		if (loop_transfer_bio(lo, new_bio, old_bio))
Linus Torvalds's avatar
Linus Torvalds committed
560
			goto err;
Linus Torvalds's avatar
Linus Torvalds committed
561 562
	}

563
	generic_make_request(new_bio);
Linus Torvalds's avatar
Linus Torvalds committed
564
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
565

Linus Torvalds's avatar
Linus Torvalds committed
566
err:
Linus Torvalds's avatar
Linus Torvalds committed
567 568
	if (atomic_dec_and_test(&lo->lo_pending))
		up(&lo->lo_bh_mutex);
569
	loop_put_buffer(new_bio);
Linus Torvalds's avatar
Linus Torvalds committed
570
out:
571
	bio_io_error(old_bio, old_bio->bi_size);
Linus Torvalds's avatar
Linus Torvalds committed
572 573 574 575 576 577
	return 0;
inactive:
	spin_unlock_irq(&lo->lo_lock);
	goto out;
}

Linus Torvalds's avatar
Linus Torvalds committed
578
static inline void loop_handle_bio(struct loop_device *lo, struct bio *bio)
Linus Torvalds's avatar
Linus Torvalds committed
579 580 581 582 583 584 585
{
	int ret;

	/*
	 * For block backed loop, we know this is a READ
	 */
	if (lo->lo_flags & LO_FLAGS_DO_BMAP) {
Linus Torvalds's avatar
Linus Torvalds committed
586
		ret = do_bio_filebacked(lo, bio);
587
		bio_endio(bio, bio->bi_size, ret);
Linus Torvalds's avatar
Linus Torvalds committed
588
	} else {
Linus Torvalds's avatar
Linus Torvalds committed
589
		struct bio *rbh = bio->bi_private;
Linus Torvalds's avatar
Linus Torvalds committed
590

591
		ret = loop_transfer_bio(lo, bio, rbh);
Linus Torvalds's avatar
Linus Torvalds committed
592

593
		bio_endio(rbh, rbh->bi_size, ret);
Linus Torvalds's avatar
Linus Torvalds committed
594
		loop_put_buffer(bio);
Linus Torvalds's avatar
Linus Torvalds committed
595
	}
Linus Torvalds's avatar
Linus Torvalds committed
596
}
Linus Torvalds's avatar
Linus Torvalds committed
597

Linus Torvalds's avatar
Linus Torvalds committed
598 599 600 601 602 603 604 605 606
/*
 * worker thread that handles reads/writes to file backed loop devices,
 * to avoid blocking in our make_request_fn. it also does loop decrypting
 * on reads for block backed loop, as that is too heavy to do from
 * b_end_io context where irqs may be disabled.
 */
static int loop_thread(void *data)
{
	struct loop_device *lo = data;
Linus Torvalds's avatar
Linus Torvalds committed
607
	struct bio *bio;
Linus Torvalds's avatar
Linus Torvalds committed
608

609
	daemonize("loop%d", lo->lo_number);
Linus Torvalds's avatar
Linus Torvalds committed
610

Andries E. Brouwer's avatar
Andries E. Brouwer committed
611 612 613 614 615 616
	/*
	 * loop can be used in an encrypted device,
	 * hence, it mustn't be stopped at all
	 * because it could be indirectly used during suspension
	 */
	current->flags |= PF_IOTHREAD;
Linus Torvalds's avatar
Linus Torvalds committed
617

Linus Torvalds's avatar
Linus Torvalds committed
618
	set_user_nice(current, -20);
Linus Torvalds's avatar
Linus Torvalds committed
619 620 621 622 623 624 625 626 627 628 629

	lo->lo_state = Lo_bound;
	atomic_inc(&lo->lo_pending);

	/*
	 * up sem, we are running
	 */
	up(&lo->lo_sem);

	for (;;) {
		down_interruptible(&lo->lo_bh_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
630 631 632 633
		/*
		 * could be upped because of tear-down, not because of
		 * pending work
		 */
Linus Torvalds's avatar
Linus Torvalds committed
634 635 636
		if (!atomic_read(&lo->lo_pending))
			break;

Linus Torvalds's avatar
Linus Torvalds committed
637 638 639
		bio = loop_get_bio(lo);
		if (!bio) {
			printk("loop: missing bio\n");
Linus Torvalds's avatar
Linus Torvalds committed
640
			continue;
Linus Torvalds's avatar
Linus Torvalds committed
641
		}
Linus Torvalds's avatar
Linus Torvalds committed
642
		loop_handle_bio(lo, bio);
Linus Torvalds's avatar
Linus Torvalds committed
643 644 645 646 647 648 649

		/*
		 * upped both for pending work and tear-down, lo_pending
		 * will hit zero then
		 */
		if (atomic_dec_and_test(&lo->lo_pending))
			break;
Linus Torvalds's avatar
Linus Torvalds committed
650
	}
Linus Torvalds's avatar
Linus Torvalds committed
651 652 653

	up(&lo->lo_sem);
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
654 655
}

656 657
static int loop_set_fd(struct loop_device *lo, struct file *lo_file,
		       struct block_device *bdev, unsigned int arg)
Linus Torvalds's avatar
Linus Torvalds committed
658 659 660
{
	struct file	*file;
	struct inode	*inode;
Alexander Viro's avatar
Alexander Viro committed
661
	struct block_device *lo_device = NULL;
662
	struct address_space *mapping;
Alexander Viro's avatar
Alexander Viro committed
663
	unsigned lo_blocksize;
Linus Torvalds's avatar
Linus Torvalds committed
664 665
	int		lo_flags = 0;
	int		error;
Linus Torvalds's avatar
Linus Torvalds committed
666

667 668
	/* This is safe, since we have a reference from open(). */
	__module_get(THIS_MODULE);
Linus Torvalds's avatar
Linus Torvalds committed
669 670

	error = -EBUSY;
Linus Torvalds's avatar
Linus Torvalds committed
671
	if (lo->lo_state != Lo_unbound)
Linus Torvalds's avatar
Linus Torvalds committed
672
		goto out;
Linus Torvalds's avatar
Linus Torvalds committed
673

Linus Torvalds's avatar
Linus Torvalds committed
674 675 676 677 678
	error = -EBADF;
	file = fget(arg);
	if (!file)
		goto out;

679 680
	mapping = file->f_mapping;
	inode = mapping->host;
Linus Torvalds's avatar
Linus Torvalds committed
681

Linus Torvalds's avatar
Linus Torvalds committed
682 683
	if (!(file->f_mode & FMODE_WRITE))
		lo_flags |= LO_FLAGS_READ_ONLY;
Linus Torvalds's avatar
Linus Torvalds committed
684

Linus Torvalds's avatar
Linus Torvalds committed
685
	if (S_ISBLK(inode->i_mode)) {
686 687
		lo_device = inode->i_bdev;
		if (lo_device == bdev) {
Linus Torvalds's avatar
Linus Torvalds committed
688
			error = -EBUSY;
689
			goto out_putf;
Linus Torvalds's avatar
Linus Torvalds committed
690
		}
Alexander Viro's avatar
Alexander Viro committed
691 692 693
		lo_blocksize = block_size(lo_device);
		if (bdev_read_only(lo_device))
			lo_flags |= LO_FLAGS_READ_ONLY;
Linus Torvalds's avatar
Linus Torvalds committed
694
	} else if (S_ISREG(inode->i_mode)) {
695
		struct address_space_operations *aops = mapping->a_ops;
Linus Torvalds's avatar
Linus Torvalds committed
696 697 698 699
		/*
		 * If we can't read - sorry. If we only can't write - well,
		 * it's going to be read-only.
		 */
700
		error = -EINVAL;
701
		if (!inode->i_fop->sendfile)
Linus Torvalds's avatar
Linus Torvalds committed
702 703 704
			goto out_putf;

		if (!aops->prepare_write || !aops->commit_write)
Linus Torvalds's avatar
Linus Torvalds committed
705
			lo_flags |= LO_FLAGS_READ_ONLY;
Linus Torvalds's avatar
Linus Torvalds committed
706

707
		lo_blocksize = inode->i_blksize;
Linus Torvalds's avatar
Linus Torvalds committed
708 709 710 711
		lo_flags |= LO_FLAGS_DO_BMAP;
		error = 0;
	} else
		goto out_putf;
Linus Torvalds's avatar
Linus Torvalds committed
712

Alexander Viro's avatar
Alexander Viro committed
713
	if (!(lo_file->f_mode & FMODE_WRITE))
Linus Torvalds's avatar
Linus Torvalds committed
714
		lo_flags |= LO_FLAGS_READ_ONLY;
Linus Torvalds's avatar
Linus Torvalds committed
715

716
	set_device_ro(bdev, (lo_flags & LO_FLAGS_READ_ONLY) != 0);
Linus Torvalds's avatar
Linus Torvalds committed
717

Alexander Viro's avatar
Alexander Viro committed
718
	lo->lo_blocksize = lo_blocksize;
Linus Torvalds's avatar
Linus Torvalds committed
719 720 721 722 723
	lo->lo_device = lo_device;
	lo->lo_flags = lo_flags;
	lo->lo_backing_file = file;
	lo->transfer = NULL;
	lo->ioctl = NULL;
724
	lo->lo_sizelimit = 0;
725 726 727 728
	if (figure_loop_size(lo)) {
		error = -EFBIG;
		goto out_putf;
	}
729 730
	lo->old_gfp_mask = mapping_gfp_mask(mapping);
	mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
Linus Torvalds's avatar
Linus Torvalds committed
731

Linus Torvalds's avatar
Linus Torvalds committed
732
	lo->lo_bio = lo->lo_biotail = NULL;
733 734 735 736 737

	/*
	 * set queue make_request_fn, and add limits based on lower level
	 * device
	 */
738 739
	blk_queue_make_request(lo->lo_queue, loop_make_request);
	lo->lo_queue->queuedata = lo;
740 741 742 743 744 745 746

	/*
	 * we remap to a block device, make sure we correctly stack limits
	 */
	if (S_ISBLK(inode->i_mode)) {
		request_queue_t *q = bdev_get_queue(lo_device);

747 748 749
		blk_queue_max_sectors(lo->lo_queue, q->max_sectors);
		blk_queue_max_phys_segments(lo->lo_queue,q->max_phys_segments);
		blk_queue_max_hw_segments(lo->lo_queue, q->max_hw_segments);
750
		blk_queue_hardsect_size(lo->lo_queue, queue_hardsect_size(q));
751 752 753
		blk_queue_max_segment_size(lo->lo_queue, q->max_segment_size);
		blk_queue_segment_boundary(lo->lo_queue, q->seg_boundary_mask);
		blk_queue_merge_bvec(lo->lo_queue, q->merge_bvec_fn);
754 755
	}

756 757
	set_blocksize(bdev, lo_blocksize);

Albert Cahalan's avatar
Albert Cahalan committed
758
	kernel_thread(loop_thread, lo, CLONE_KERNEL);
Linus Torvalds's avatar
Linus Torvalds committed
759 760
	down(&lo->lo_sem);
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
761 762 763 764

 out_putf:
	fput(file);
 out:
765 766
	/* This is safe: open() is still holding a reference. */
	module_put(THIS_MODULE);
Linus Torvalds's avatar
Linus Torvalds committed
767 768 769
	return error;
}

Andries E. Brouwer's avatar
Andries E. Brouwer committed
770 771
static int
loop_release_xfer(struct loop_device *lo)
Linus Torvalds's avatar
Linus Torvalds committed
772
{
Andries E. Brouwer's avatar
Andries E. Brouwer committed
773 774 775 776 777 778 779 780 781
	int err = 0;
	struct loop_func_table *xfer = lo->lo_encryption;

	if (xfer) {
		if (xfer->release)
			err = xfer->release(lo);
		lo->transfer = NULL;
		lo->lo_encryption = NULL;
		module_put(xfer->owner);
Linus Torvalds's avatar
Linus Torvalds committed
782 783 784 785
	}
	return err;
}

786
static int
Andries E. Brouwer's avatar
Andries E. Brouwer committed
787 788
loop_init_xfer(struct loop_device *lo, struct loop_func_table *xfer,
	       const struct loop_info64 *i)
Linus Torvalds's avatar
Linus Torvalds committed
789
{
Andries E. Brouwer's avatar
Andries E. Brouwer committed
790 791 792 793 794 795 796
	int err = 0;

	if (xfer) {
		struct module *owner = xfer->owner;

		if (!try_module_get(owner))
			return -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
797 798
		if (xfer->init)
			err = xfer->init(lo, i);
Andries E. Brouwer's avatar
Andries E. Brouwer committed
799 800 801 802
		if (err)
			module_put(owner);
		else
			lo->lo_encryption = xfer;
Linus Torvalds's avatar
Linus Torvalds committed
803 804
	}
	return err;
Andries E. Brouwer's avatar
Andries E. Brouwer committed
805
}
Linus Torvalds's avatar
Linus Torvalds committed
806

Linus Torvalds's avatar
Linus Torvalds committed
807
static int loop_clr_fd(struct loop_device *lo, struct block_device *bdev)
Linus Torvalds's avatar
Linus Torvalds committed
808
{
Linus Torvalds's avatar
Linus Torvalds committed
809 810
	struct file *filp = lo->lo_backing_file;
	int gfp = lo->old_gfp_mask;
Linus Torvalds's avatar
Linus Torvalds committed
811

Linus Torvalds's avatar
Linus Torvalds committed
812
	if (lo->lo_state != Lo_bound)
Linus Torvalds's avatar
Linus Torvalds committed
813
		return -ENXIO;
Andries E. Brouwer's avatar
Andries E. Brouwer committed
814

Linus Torvalds's avatar
Linus Torvalds committed
815 816
	if (lo->lo_refcnt > 1)	/* we needed one fd for the ioctl */
		return -EBUSY;
Andries E. Brouwer's avatar
Andries E. Brouwer committed
817 818

	if (filp == NULL)
Linus Torvalds's avatar
Linus Torvalds committed
819
		return -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
820

Linus Torvalds's avatar
Linus Torvalds committed
821 822 823 824 825
	spin_lock_irq(&lo->lo_lock);
	lo->lo_state = Lo_rundown;
	if (atomic_dec_and_test(&lo->lo_pending))
		up(&lo->lo_bh_mutex);
	spin_unlock_irq(&lo->lo_lock);
Linus Torvalds's avatar
Linus Torvalds committed
826

Linus Torvalds's avatar
Linus Torvalds committed
827
	down(&lo->lo_sem);
Linus Torvalds's avatar
Linus Torvalds committed
828

Linus Torvalds's avatar
Linus Torvalds committed
829
	lo->lo_backing_file = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
830 831 832 833

	loop_release_xfer(lo);
	lo->transfer = NULL;
	lo->ioctl = NULL;
834
	lo->lo_device = NULL;
Andries E. Brouwer's avatar
Andries E. Brouwer committed
835
	lo->lo_encryption = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
836
	lo->lo_offset = 0;
837
	lo->lo_sizelimit = 0;
Linus Torvalds's avatar
Linus Torvalds committed
838
	lo->lo_encrypt_key_size = 0;
Linus Torvalds's avatar
Linus Torvalds committed
839
	lo->lo_flags = 0;
Linus Torvalds's avatar
Linus Torvalds committed
840
	memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE);
Andries E. Brouwer's avatar
Andries E. Brouwer committed
841 842
	memset(lo->lo_crypt_name, 0, LO_NAME_SIZE);
	memset(lo->lo_file_name, 0, LO_NAME_SIZE);
Linus Torvalds's avatar
Linus Torvalds committed
843
	invalidate_bdev(bdev, 0);
844
	set_capacity(disks[lo->lo_number], 0);
845
	mapping_set_gfp_mask(filp->f_mapping, gfp);
Linus Torvalds's avatar
Linus Torvalds committed
846 847
	lo->lo_state = Lo_unbound;
	fput(filp);
848 849
	/* This is safe: open() is still holding a reference. */
	module_put(THIS_MODULE);
Linus Torvalds's avatar
Linus Torvalds committed
850 851 852
	return 0;
}

853 854
static int
loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
Linus Torvalds's avatar
Linus Torvalds committed
855 856
{
	int err;
Andries E. Brouwer's avatar
Andries E. Brouwer committed
857
	struct loop_func_table *xfer;
Linus Torvalds's avatar
Linus Torvalds committed
858

Andries E. Brouwer's avatar
Andries E. Brouwer committed
859
	if (lo->lo_encrypt_key_size && lo->lo_key_owner != current->uid &&
Linus Torvalds's avatar
Linus Torvalds committed
860 861
	    !capable(CAP_SYS_ADMIN))
		return -EPERM;
Linus Torvalds's avatar
Linus Torvalds committed
862
	if (lo->lo_state != Lo_bound)
Linus Torvalds's avatar
Linus Torvalds committed
863
		return -ENXIO;
864
	if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE)
Linus Torvalds's avatar
Linus Torvalds committed
865
		return -EINVAL;
866

Linus Torvalds's avatar
Linus Torvalds committed
867
	err = loop_release_xfer(lo);
Andries E. Brouwer's avatar
Andries E. Brouwer committed
868 869
	if (err)
		return err;
870

Andries E. Brouwer's avatar
Andries E. Brouwer committed
871 872
	if (info->lo_encrypt_type) {
		unsigned int type = info->lo_encrypt_type;
873

Andries E. Brouwer's avatar
Andries E. Brouwer committed
874 875 876 877 878 879 880 881 882
		if (type >= MAX_LO_CRYPT)
			return -EINVAL;
		xfer = xfer_funcs[type];
		if (xfer == NULL)
			return -EINVAL;
	} else
		xfer = NULL;

	err = loop_init_xfer(lo, xfer, info);
Linus Torvalds's avatar
Linus Torvalds committed
883
	if (err)
Andries E. Brouwer's avatar
Andries E. Brouwer committed
884 885
		return err;

886 887
	if (lo->lo_offset != info->lo_offset ||
	    lo->lo_sizelimit != info->lo_sizelimit) {
Andries E. Brouwer's avatar
Andries E. Brouwer committed
888
		lo->lo_offset = info->lo_offset;
889
		lo->lo_sizelimit = info->lo_sizelimit;
Andries E. Brouwer's avatar
Andries E. Brouwer committed
890 891 892
		if (figure_loop_size(lo))
			return -EFBIG;
	}
Linus Torvalds's avatar
Linus Torvalds committed
893

Andries E. Brouwer's avatar
Andries E. Brouwer committed
894 895 896 897
	memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE);
	memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE);
	lo->lo_file_name[LO_NAME_SIZE-1] = 0;
	lo->lo_crypt_name[LO_NAME_SIZE-1] = 0;
Linus Torvalds's avatar
Linus Torvalds committed
898

Andries E. Brouwer's avatar
Andries E. Brouwer committed
899 900 901 902 903
	if (!xfer)
		xfer = &none_funcs;
	lo->transfer = xfer->transfer;
	lo->ioctl = xfer->ioctl;

904 905 906 907
	lo->lo_encrypt_key_size = info->lo_encrypt_key_size;
	lo->lo_init[0] = info->lo_init[0];
	lo->lo_init[1] = info->lo_init[1];
	if (info->lo_encrypt_key_size) {
Andries E. Brouwer's avatar
Andries E. Brouwer committed
908
		memcpy(lo->lo_encrypt_key, info->lo_encrypt_key,
909
		       info->lo_encrypt_key_size);
Andries E. Brouwer's avatar
Andries E. Brouwer committed
910
		lo->lo_key_owner = current->uid;
Linus Torvalds's avatar
Linus Torvalds committed
911
	}	
912

Linus Torvalds's avatar
Linus Torvalds committed
913 914 915
	return 0;
}

916 917
static int
loop_get_status(struct loop_device *lo, struct loop_info64 *info)
Linus Torvalds's avatar
Linus Torvalds committed
918
{
Linus Torvalds's avatar
Linus Torvalds committed
919
	struct file *file = lo->lo_backing_file;
Alexander Viro's avatar
Alexander Viro committed
920 921
	struct kstat stat;
	int error;
Linus Torvalds's avatar
Linus Torvalds committed
922

Linus Torvalds's avatar
Linus Torvalds committed
923
	if (lo->lo_state != Lo_bound)
Linus Torvalds's avatar
Linus Torvalds committed
924
		return -ENXIO;
Alexander Viro's avatar
Alexander Viro committed
925 926 927
	error = vfs_getattr(file->f_vfsmnt, file->f_dentry, &stat);
	if (error)
		return error;
928 929
	memset(info, 0, sizeof(*info));
	info->lo_number = lo->lo_number;
930
	info->lo_device = huge_encode_dev(stat.dev);
931
	info->lo_inode = stat.ino;
932
	info->lo_rdevice = huge_encode_dev(lo->lo_device ? stat.rdev : stat.dev);
933
	info->lo_offset = lo->lo_offset;
934
	info->lo_sizelimit = lo->lo_sizelimit;
935
	info->lo_flags = lo->lo_flags;
Andries E. Brouwer's avatar
Andries E. Brouwer committed
936 937
	memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE);
	memcpy(info->lo_crypt_name, lo->lo_crypt_name, LO_NAME_SIZE);
Andries E. Brouwer's avatar
Andries E. Brouwer committed
938 939
	info->lo_encrypt_type =
		lo->lo_encryption ? lo->lo_encryption->number : 0;
Linus Torvalds's avatar
Linus Torvalds committed
940
	if (lo->lo_encrypt_key_size && capable(CAP_SYS_ADMIN)) {
941 942
		info->lo_encrypt_key_size = lo->lo_encrypt_key_size;
		memcpy(info->lo_encrypt_key, lo->lo_encrypt_key,
Linus Torvalds's avatar
Linus Torvalds committed
943 944
		       lo->lo_encrypt_key_size);
	}
945 946 947 948 949 950
	return 0;
}

static void
loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64)
{
951
	memset(info64, 0, sizeof(*info64));
952 953 954 955 956
	info64->lo_number = info->lo_number;
	info64->lo_device = info->lo_device;
	info64->lo_inode = info->lo_inode;
	info64->lo_rdevice = info->lo_rdevice;
	info64->lo_offset = info->lo_offset;
957
	info64->lo_sizelimit = 0;
958 959 960 961 962
	info64->lo_encrypt_type = info->lo_encrypt_type;
	info64->lo_encrypt_key_size = info->lo_encrypt_key_size;
	info64->lo_flags = info->lo_flags;
	info64->lo_init[0] = info->lo_init[0];
	info64->lo_init[1] = info->lo_init[1];
Andries E. Brouwer's avatar
Andries E. Brouwer committed
963 964 965 966
	if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
		memcpy(info64->lo_crypt_name, info->lo_name, LO_NAME_SIZE);
	else
		memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE);
967 968 969 970 971 972
	memcpy(info64->lo_encrypt_key, info->lo_encrypt_key, LO_KEY_SIZE);
}

static int
loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info)
{
973
	memset(info, 0, sizeof(*info));
974 975 976 977 978 979 980 981 982 983
	info->lo_number = info64->lo_number;
	info->lo_device = info64->lo_device;
	info->lo_inode = info64->lo_inode;
	info->lo_rdevice = info64->lo_rdevice;
	info->lo_offset = info64->lo_offset;
	info->lo_encrypt_type = info64->lo_encrypt_type;
	info->lo_encrypt_key_size = info64->lo_encrypt_key_size;
	info->lo_flags = info64->lo_flags;
	info->lo_init[0] = info64->lo_init[0];
	info->lo_init[1] = info64->lo_init[1];
Andries E. Brouwer's avatar
Andries E. Brouwer committed
984 985 986 987 988
	if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
		memcpy(info->lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
	else
		memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE);
	memcpy(info->lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);
989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052

	/* error in case values were truncated */
	if (info->lo_device != info64->lo_device ||
	    info->lo_rdevice != info64->lo_rdevice ||
	    info->lo_inode != info64->lo_inode ||
	    info->lo_offset != info64->lo_offset)
		return -EOVERFLOW;

	return 0;
}

static int
loop_set_status_old(struct loop_device *lo, const struct loop_info *arg)
{
	struct loop_info info;
	struct loop_info64 info64;

	if (copy_from_user(&info, arg, sizeof (struct loop_info)))
		return -EFAULT;
	loop_info64_from_old(&info, &info64);
	return loop_set_status(lo, &info64);
}

static int
loop_set_status64(struct loop_device *lo, const struct loop_info64 *arg)
{
	struct loop_info64 info64;

	if (copy_from_user(&info64, arg, sizeof (struct loop_info64)))
		return -EFAULT;
	return loop_set_status(lo, &info64);
}

static int
loop_get_status_old(struct loop_device *lo, struct loop_info *arg) {
	struct loop_info info;
	struct loop_info64 info64;
	int err = 0;

	if (!arg)
		err = -EINVAL;
	if (!err)
		err = loop_get_status(lo, &info64);
	if (!err)
		err = loop_info64_to_old(&info64, &info);
	if (!err && copy_to_user(arg, &info, sizeof(info)))
		err = -EFAULT;

	return err;
}

static int
loop_get_status64(struct loop_device *lo, struct loop_info64 *arg) {
	struct loop_info64 info64;
	int err = 0;

	if (!arg)
		err = -EINVAL;
	if (!err)
		err = loop_get_status(lo, &info64);
	if (!err && copy_to_user(arg, &info64, sizeof(info64)))
		err = -EFAULT;

	return err;
Linus Torvalds's avatar
Linus Torvalds committed
1053 1054 1055 1056 1057
}

static int lo_ioctl(struct inode * inode, struct file * file,
	unsigned int cmd, unsigned long arg)
{
Alexander Viro's avatar
Alexander Viro committed
1058 1059
	struct loop_device *lo = inode->i_bdev->bd_disk->private_data;
	int err;
Linus Torvalds's avatar
Linus Torvalds committed
1060

Linus Torvalds's avatar
Linus Torvalds committed
1061
	down(&lo->lo_ctl_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
1062 1063
	switch (cmd) {
	case LOOP_SET_FD:
1064
		err = loop_set_fd(lo, file, inode->i_bdev, arg);
Linus Torvalds's avatar
Linus Torvalds committed
1065
		break;
Linus Torvalds's avatar
Linus Torvalds committed
1066
	case LOOP_CLR_FD:
Linus Torvalds's avatar
Linus Torvalds committed
1067
		err = loop_clr_fd(lo, inode->i_bdev);
Linus Torvalds's avatar
Linus Torvalds committed
1068
		break;
Linus Torvalds's avatar
Linus Torvalds committed
1069
	case LOOP_SET_STATUS:
1070
		err = loop_set_status_old(lo, (struct loop_info *) arg);
Linus Torvalds's avatar
Linus Torvalds committed
1071
		break;
Linus Torvalds's avatar
Linus Torvalds committed
1072
	case LOOP_GET_STATUS:
1073 1074 1075 1076 1077 1078 1079
		err = loop_get_status_old(lo, (struct loop_info *) arg);
		break;
	case LOOP_SET_STATUS64:
		err = loop_set_status64(lo, (struct loop_info64 *) arg);
		break;
	case LOOP_GET_STATUS64:
		err = loop_get_status64(lo, (struct loop_info64 *) arg);
Linus Torvalds's avatar
Linus Torvalds committed
1080
		break;
Linus Torvalds's avatar
Linus Torvalds committed
1081
	default:
Linus Torvalds's avatar
Linus Torvalds committed
1082
		err = lo->ioctl ? lo->ioctl(lo, cmd, arg) : -EINVAL;
Linus Torvalds's avatar
Linus Torvalds committed
1083
	}
Linus Torvalds's avatar
Linus Torvalds committed
1084 1085
	up(&lo->lo_ctl_mutex);
	return err;
Linus Torvalds's avatar
Linus Torvalds committed
1086 1087 1088 1089
}

static int lo_open(struct inode *inode, struct file *file)
{
Alexander Viro's avatar
Alexander Viro committed
1090
	struct loop_device *lo = inode->i_bdev->bd_disk->private_data;
Linus Torvalds's avatar
Linus Torvalds committed
1091 1092

	down(&lo->lo_ctl_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
1093
	lo->lo_refcnt++;
Linus Torvalds's avatar
Linus Torvalds committed
1094
	up(&lo->lo_ctl_mutex);
Andries E. Brouwer's avatar
Andries E. Brouwer committed
1095

Linus Torvalds's avatar
Linus Torvalds committed
1096 1097 1098 1099 1100
	return 0;
}

static int lo_release(struct inode *inode, struct file *file)
{
Alexander Viro's avatar
Alexander Viro committed
1101
	struct loop_device *lo = inode->i_bdev->bd_disk->private_data;
Linus Torvalds's avatar
Linus Torvalds committed
1102 1103 1104 1105

	down(&lo->lo_ctl_mutex);
	--lo->lo_refcnt;
	up(&lo->lo_ctl_mutex);
Andries E. Brouwer's avatar
Andries E. Brouwer committed
1106

Linus Torvalds's avatar
Linus Torvalds committed
1107 1108 1109 1110
	return 0;
}

static struct block_device_operations lo_fops = {
1111 1112 1113 1114
	.owner =	THIS_MODULE,
	.open =		lo_open,
	.release =	lo_release,
	.ioctl =	lo_ioctl,
Linus Torvalds's avatar
Linus Torvalds committed
1115 1116 1117 1118 1119 1120
};

/*
 * And now the modules code and kernel interface.
 */
MODULE_PARM(max_loop, "i");
1121
MODULE_PARM_DESC(max_loop, "Maximum number of loop devices (1-256)");
Linus Torvalds's avatar
Linus Torvalds committed
1122
MODULE_LICENSE("GPL");
Linus Torvalds's avatar
Linus Torvalds committed
1123 1124 1125

int loop_register_transfer(struct loop_func_table *funcs)
{
Andries E. Brouwer's avatar
Andries E. Brouwer committed
1126 1127 1128
	unsigned int n = funcs->number;

	if (n >= MAX_LO_CRYPT || xfer_funcs[n])
Linus Torvalds's avatar
Linus Torvalds committed
1129
		return -EINVAL;
Andries E. Brouwer's avatar
Andries E. Brouwer committed
1130 1131
	xfer_funcs[n] = funcs;
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
1132 1133 1134 1135
}

int loop_unregister_transfer(int number)
{
Andries E. Brouwer's avatar
Andries E. Brouwer committed
1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151
	unsigned int n = number;
	struct loop_device *lo;
	struct loop_func_table *xfer;

	if (n == 0 || n >= MAX_LO_CRYPT || (xfer = xfer_funcs[n]) == NULL)
		return -EINVAL;

	xfer_funcs[n] = NULL;

	for (lo = &loop_dev[0]; lo < &loop_dev[max_loop]; lo++) {
		down(&lo->lo_ctl_mutex);

		if (lo->lo_encryption == xfer)
			loop_release_xfer(lo);

		up(&lo->lo_ctl_mutex);
Linus Torvalds's avatar
Linus Torvalds committed
1152
	}
Andries E. Brouwer's avatar
Andries E. Brouwer committed
1153 1154

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
1155 1156 1157 1158 1159
}

EXPORT_SYMBOL(loop_register_transfer);
EXPORT_SYMBOL(loop_unregister_transfer);

Andries E. Brouwer's avatar
Andries E. Brouwer committed
1160
int __init loop_init(void)
Linus Torvalds's avatar
Linus Torvalds committed
1161 1162 1163
{
	int	i;

1164
	if (max_loop < 1 || max_loop > 256) {
Linus Torvalds's avatar
Linus Torvalds committed
1165
		printk(KERN_WARNING "loop: invalid max_loop (must be between"
1166
				    " 1 and 256), using default (8)\n");
Linus Torvalds's avatar
Linus Torvalds committed
1167 1168 1169
		max_loop = 8;
	}

1170
	if (register_blkdev(LOOP_MAJOR, "loop"))
Linus Torvalds's avatar
Linus Torvalds committed
1171 1172 1173 1174
		return -EIO;

	loop_dev = kmalloc(max_loop * sizeof(struct loop_device), GFP_KERNEL);
	if (!loop_dev)
1175
		goto out_mem1;
Andrew Morton's avatar
Andrew Morton committed
1176
	memset(loop_dev, 0, max_loop * sizeof(struct loop_device));
Linus Torvalds's avatar
Linus Torvalds committed
1177

1178
	disks = kmalloc(max_loop * sizeof(struct gendisk *), GFP_KERNEL);
Alexander Viro's avatar
Alexander Viro committed
1179
	if (!disks)
1180
		goto out_mem2;
Linus Torvalds's avatar
Linus Torvalds committed
1181

1182
	for (i = 0; i < max_loop; i++) {
1183
		disks[i] = alloc_disk(1);
1184
		if (!disks[i])
1185
			goto out_mem3;
1186 1187
	}

1188 1189
	devfs_mk_dir("loop");

Linus Torvalds's avatar
Linus Torvalds committed
1190 1191
	for (i = 0; i < max_loop; i++) {
		struct loop_device *lo = &loop_dev[i];
1192
		struct gendisk *disk = disks[i];
Andrew Morton's avatar
Andrew Morton committed
1193

1194
		memset(lo, 0, sizeof(*lo));
Andrew Morton's avatar
Andrew Morton committed
1195 1196 1197 1198
		lo->lo_queue = blk_alloc_queue(GFP_KERNEL);
		if (!lo->lo_queue)
			goto out_mem4;
		disks[i]->queue = lo->lo_queue;
Linus Torvalds's avatar
Linus Torvalds committed
1199 1200 1201 1202 1203
		init_MUTEX(&lo->lo_ctl_mutex);
		init_MUTEX_LOCKED(&lo->lo_sem);
		init_MUTEX_LOCKED(&lo->lo_bh_mutex);
		lo->lo_number = i;
		spin_lock_init(&lo->lo_lock);
Alexander Viro's avatar
Alexander Viro committed
1204 1205 1206
		disk->major = LOOP_MAJOR;
		disk->first_minor = i;
		disk->fops = &lo_fops;
1207
		sprintf(disk->disk_name, "loop%d", i);
Christoph Hellwig's avatar
Christoph Hellwig committed
1208
		sprintf(disk->devfs_name, "loop/%d", i);
Alexander Viro's avatar
Alexander Viro committed
1209
		disk->private_data = lo;
1210
		disk->queue = lo->lo_queue;
Alexander Viro's avatar
Alexander Viro committed
1211
		add_disk(disk);
Linus Torvalds's avatar
Linus Torvalds committed
1212
	}
Linus Torvalds's avatar
Linus Torvalds committed
1213
	printk(KERN_INFO "loop: loaded (max %d devices)\n", max_loop);
Linus Torvalds's avatar
Linus Torvalds committed
1214
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
1215

Andrew Morton's avatar
Andrew Morton committed
1216 1217 1218 1219
out_mem4:
	while (i--)
		blk_put_queue(loop_dev[i].lo_queue);
	i = max_loop;
1220
out_mem3:
1221 1222
	while (i--)
		put_disk(disks[i]);
Alexander Viro's avatar
Alexander Viro committed
1223
	kfree(disks);
1224
out_mem2:
Linus Torvalds's avatar
Linus Torvalds committed
1225
	kfree(loop_dev);
1226 1227
out_mem1:
	unregister_blkdev(LOOP_MAJOR, "loop");
Linus Torvalds's avatar
Linus Torvalds committed
1228 1229
	printk(KERN_ERR "loop: ran out of memory\n");
	return -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
1230 1231
}

Andries E. Brouwer's avatar
Andries E. Brouwer committed
1232
void loop_exit(void)
Linus Torvalds's avatar
Linus Torvalds committed
1233
{
Alexander Viro's avatar
Alexander Viro committed
1234
	int i;
Andries E. Brouwer's avatar
Andries E. Brouwer committed
1235

1236 1237
	for (i = 0; i < max_loop; i++) {
		del_gendisk(disks[i]);
1238
		blk_put_queue(loop_dev[i].lo_queue);
1239 1240
		put_disk(disks[i]);
	}
Alexander Viro's avatar
Alexander Viro committed
1241
	devfs_remove("loop");
1242
	if (unregister_blkdev(LOOP_MAJOR, "loop"))
Linus Torvalds's avatar
Linus Torvalds committed
1243 1244
		printk(KERN_WARNING "loop: cannot unregister blkdev\n");

Alexander Viro's avatar
Alexander Viro committed
1245
	kfree(disks);
Linus Torvalds's avatar
Linus Torvalds committed
1246
	kfree(loop_dev);
Linus Torvalds's avatar
Linus Torvalds committed
1247
}
Linus Torvalds's avatar
Linus Torvalds committed
1248 1249 1250

module_init(loop_init);
module_exit(loop_exit);
Linus Torvalds's avatar
Linus Torvalds committed
1251 1252 1253 1254

#ifndef MODULE
static int __init max_loop_setup(char *str)
{
Linus Torvalds's avatar
Linus Torvalds committed
1255
	max_loop = simple_strtol(str, NULL, 0);
Linus Torvalds's avatar
Linus Torvalds committed
1256 1257 1258 1259 1260
	return 1;
}

__setup("max_loop=", max_loop_setup);
#endif