ceph: fix session locking in handle_caps, ceph_check_caps

Passing a session pointer to ceph_check_caps() used to mean it would leave
the session mutex locked.  That wasn't always possible if it wasn't passed
CHECK_CAPS_AUTHONLY.   If could unlock the passed session and lock a
differet session mutex, which was clearly wrong, and also emitted a
warning when it a racing CPU retook it and we did an unlock from the wrong
context.

This was only a problem when there was more than one MDS.

First, make ceph_check_caps unconditionally drop the session mutex, so that
it is free to lock other sessions as needed.  Then adjust the one caller
that passes in a session (handle_cap_grant) accordingly.
Signed-off-by: Sage Weil <sage@newdream.net>

fs/ceph/caps.c

@@ -1407,6 +1407,7 @@ static int try_nonblocking_invalidate(struct inode *inode)
  */
 void ceph_check_caps(struct ceph_inode_info *ci, int flags,
 		     struct ceph_mds_session *session)
+	__releases(session->s_mutex)
 {
 	struct ceph_client *client = ceph_inode_to_client(&ci->vfs_inode);
 	struct ceph_mds_client *mdsc = &client->mdsc;
@@ -1414,7 +1415,6 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags,
 	struct ceph_cap *cap;
 	int file_wanted, used;
 	int took_snap_rwsem = 0;             /* true if mdsc->snap_rwsem held */
-	int drop_session_lock = session ? 0 : 1;
 	int issued, implemented, want, retain, revoking, flushing = 0;
 	int mds = -1;   /* keep track of how far we've gone through i_caps list
 			   to avoid an infinite loop on retry */
@@ -1639,7 +1639,7 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags,
 	if (queue_invalidate)
 		ceph_queue_invalidate(inode);
 
-	if (session && drop_session_lock)
+	if (session)
 		mutex_unlock(&session->s_mutex);
 	if (took_snap_rwsem)
 		up_read(&mdsc->snap_rwsem);
@@ -2688,14 +2688,17 @@ void ceph_handle_caps(struct ceph_mds_session *session,
 	case CEPH_CAP_OP_REVOKE:
 	case CEPH_CAP_OP_GRANT:
 		r = handle_cap_grant(inode, h, session, cap, msg->middle);
-		if (r == 1)
+		if (r == 1) {
 			ceph_check_caps(ceph_inode(inode),
 					CHECK_CAPS_NODELAY|CHECK_CAPS_AUTHONLY,
 					session);
-		else if (r == 2)
+			session = NULL;
+		} else if (r == 2) {
 			ceph_check_caps(ceph_inode(inode),
 					CHECK_CAPS_NODELAY,
 					session);
+			session = NULL;
+		}
 		break;
 
 	case CEPH_CAP_OP_FLUSH_ACK:
@@ -2713,6 +2716,7 @@ void ceph_handle_caps(struct ceph_mds_session *session,
 	}
 
 done:
+	if (session)
 		mutex_unlock(&session->s_mutex);
 
 	if (check_caps)