drbd: Replace and remove old primitives

Centralize sock->mutex locking and unlocking in [drbd|conn]_prepare_command()
and [drbd|conn]_send_comman().

Therefore all *_send_* functions are touched to use these primitives instead
of drbd_get_data_sock()/drbd_put_data_sock() and former helper functions.

That change makes the *_send_* functions more standardized.
Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>

drivers/block/drbd/drbd_int.h

@@ -1060,22 +1060,6 @@ static inline struct drbd_conf *vnr_to_mdev(struct drbd_tconn *tconn, int vnr)
 	return (struct drbd_conf *)idr_find(&tconn->volumes, vnr);
 }
 
-static inline int drbd_get_data_sock(struct drbd_tconn *tconn)
-{
-	mutex_lock(&tconn->data.mutex);
-	if (!tconn->data.socket) {
-		/* Disconnected.  */
-		mutex_unlock(&tconn->data.mutex);
-		return -EIO;
-	}
-	return 0;
-}
-
-static inline void drbd_put_data_sock(struct drbd_tconn *tconn)
-{
-	mutex_unlock(&tconn->data.mutex);
-}
-
 /*
  * function declarations
  *************************/
@@ -1118,13 +1102,6 @@ extern int _conn_send_state_req(struct drbd_tconn *, int vnr, enum drbd_packet c
 				union drbd_state, union drbd_state);
 extern int _drbd_send_state(struct drbd_conf *mdev);
 extern int drbd_send_state(struct drbd_conf *mdev);
-extern int _conn_send_cmd(struct drbd_tconn *tconn, int vnr, struct drbd_socket *sock,
-			  enum drbd_packet cmd, struct p_header *h, size_t size,
-			  unsigned msg_flags);
-extern int conn_send_cmd(struct drbd_tconn *tconn, int vnr, struct drbd_socket *sock,
-			 enum drbd_packet cmd, struct p_header *h, size_t size);
-extern int conn_send_cmd2(struct drbd_tconn *tconn, enum drbd_packet cmd,
-			  char *data, size_t size);
 extern int drbd_send_sync_param(struct drbd_conf *mdev);
 extern void drbd_send_b_ack(struct drbd_conf *mdev, u32 barrier_nr,
 			    u32 set_size);
@@ -1149,7 +1126,7 @@ extern int drbd_send_ov_request(struct drbd_conf *mdev,sector_t sector,int size)
 
 extern int drbd_send_bitmap(struct drbd_conf *mdev);
 extern void drbd_send_sr_reply(struct drbd_conf *mdev, enum drbd_state_rv retcode);
-extern int conn_send_sr_reply(struct drbd_tconn *tconn, enum drbd_state_rv retcode);
+extern void conn_send_sr_reply(struct drbd_tconn *tconn, enum drbd_state_rv retcode);
 extern void drbd_free_bc(struct drbd_backing_dev *ldev);
 extern void drbd_mdev_cleanup(struct drbd_conf *mdev);
 void drbd_print_uuids(struct drbd_conf *mdev, const char *text);
@@ -1885,26 +1862,6 @@ static inline void request_ping(struct drbd_tconn *tconn)
 	wake_asender(tconn);
 }
 
-static inline int _drbd_send_cmd(struct drbd_conf *mdev, struct drbd_socket *sock,
-				  enum drbd_packet cmd, struct p_header *h, size_t size,
-				  unsigned msg_flags)
-{
-	return _conn_send_cmd(mdev->tconn, mdev->vnr, sock, cmd, h, size, msg_flags);
-}
-
-static inline int drbd_send_cmd(struct drbd_conf *mdev, struct drbd_socket *sock,
-				enum drbd_packet cmd, struct p_header *h, size_t size)
-{
-	return conn_send_cmd(mdev->tconn, mdev->vnr, sock, cmd, h, size);
-}
-
-static inline int drbd_send_short_cmd(struct drbd_conf *mdev,
-				      enum drbd_packet cmd)
-{
-	struct p_header h;
-	return drbd_send_cmd(mdev, &mdev->tconn->data, cmd, &h, sizeof(h));
-}
-
 extern void *conn_prepare_command(struct drbd_tconn *, struct drbd_socket *);
 extern void *drbd_prepare_command(struct drbd_conf *, struct drbd_socket *);
 extern int conn_send_command(struct drbd_tconn *, struct drbd_socket *,
@@ -1916,19 +1873,8 @@ extern int drbd_send_command(struct drbd_conf *, struct drbd_socket *,
 
 extern int drbd_send_ping(struct drbd_tconn *tconn);
 extern int drbd_send_ping_ack(struct drbd_tconn *tconn);
-
-static inline int drbd_send_state_req(struct drbd_conf *mdev,
-				      union drbd_state mask, union drbd_state val)
-{
-	return _conn_send_state_req(mdev->tconn, mdev->vnr, P_STATE_CHG_REQ, mask, val);
-}
-
-static inline int conn_send_state_req(struct drbd_tconn *tconn,
-				      union drbd_state mask, union drbd_state val)
-{
-	enum drbd_packet cmd = tconn->agreed_pro_version < 100 ? P_STATE_CHG_REQ : P_CONN_ST_CHG_REQ;
-	return _conn_send_state_req(tconn, 0, cmd, mask, val);
-}
+extern int drbd_send_state_req(struct drbd_conf *, union drbd_state, union drbd_state);
+extern int conn_send_state_req(struct drbd_tconn *, union drbd_state, union drbd_state);
 
 static inline void drbd_thread_stop(struct drbd_thread *thi)
 {

drivers/block/drbd/drbd_receiver.c

@@ -729,24 +729,32 @@ static struct socket *drbd_wait_for_connect(struct drbd_tconn *tconn)
 	return s_estab;
 }
 
-static int drbd_send_fp(struct drbd_tconn *tconn, struct drbd_socket *sock, enum drbd_packet cmd)
-{
-	struct p_header *h = tconn->data.sbuf;
+static int decode_header(struct drbd_tconn *, struct p_header *, struct packet_info *);
 
-	return !_conn_send_cmd(tconn, 0, sock, cmd, h, sizeof(*h), 0);
+static int send_first_packet(struct drbd_tconn *tconn, struct drbd_socket *sock,
+			     enum drbd_packet cmd)
+{
+	if (!conn_prepare_command(tconn, sock))
+		return -EIO;
+	return conn_send_command(tconn, sock, cmd, sizeof(struct p_header), NULL, 0);
 }
 
-static enum drbd_packet drbd_recv_fp(struct drbd_tconn *tconn, struct socket *sock)
+static int receive_first_packet(struct drbd_tconn *tconn, struct socket *sock)
 {
-	struct p_header80 h;
-	int rr;
-
-	rr = drbd_recv_short(sock, &h, sizeof(h), 0);
-
-	if (rr == sizeof(h) && h.magic == cpu_to_be32(DRBD_MAGIC))
-		return be16_to_cpu(h.command);
+	unsigned int header_size = drbd_header_size(tconn);
+	struct packet_info pi;
+	int err;
 
-	return 0xffff;
+	err = drbd_recv_short(sock, tconn->data.rbuf, header_size, 0);
+	if (err != header_size) {
+		if (err >= 0)
+			err = -EIO;
+		return err;
+	}
+	err = decode_header(tconn, tconn->data.rbuf, &pi);
+	if (err)
+		return err;
+	return pi.cmd;
 }
 
 /**
@@ -834,10 +842,10 @@ static int drbd_connect(struct drbd_tconn *tconn)
 		if (s) {
 			if (!tconn->data.socket) {
 				tconn->data.socket = s;
-				drbd_send_fp(tconn, &tconn->data, P_INITIAL_DATA);
+				send_first_packet(tconn, &tconn->data, P_INITIAL_DATA);
 			} else if (!tconn->meta.socket) {
 				tconn->meta.socket = s;
-				drbd_send_fp(tconn, &tconn->meta, P_INITIAL_META);
+				send_first_packet(tconn, &tconn->meta, P_INITIAL_META);
 			} else {
 				conn_err(tconn, "Logic error in drbd_connect()\n");
 				goto out_release_sockets;
@@ -855,7 +863,7 @@ static int drbd_connect(struct drbd_tconn *tconn)
 retry:
 		s = drbd_wait_for_connect(tconn);
 		if (s) {
-			try = drbd_recv_fp(tconn, s);
+			try = receive_first_packet(tconn, s);
 			drbd_socket_okay(&tconn->data.socket);
 			drbd_socket_okay(&tconn->meta.socket);
 			switch (try) {
@@ -1324,6 +1332,10 @@ read_in_block(struct drbd_conf *mdev, u64 id, sector_t sector,
 		crypto_hash_digestsize(mdev->tconn->integrity_r_tfm) : 0;
 
 	if (dgs) {
+		/*
+		 * FIXME: Receive the incoming digest into the receive buffer
+		 *	  here, together with its struct p_data?
+		 */
 		err = drbd_recv_all_warn(mdev->tconn, dig_in, dgs);
 		if (err)
 			return NULL;
@@ -4019,8 +4031,8 @@ static void drbdd(struct drbd_tconn *tconn)
 
 		err = cmd->fn(tconn, &pi);
 		if (err) {
-			conn_err(tconn, "error receiving %s, l: %d!\n",
-			    cmdname(pi.cmd), pi.size);
+			conn_err(tconn, "error receiving %s, e: %d l: %d!\n",
+				 cmdname(pi.cmd), err, pi.size);
 			goto err_out;
 		}
 	}
@@ -4179,27 +4191,17 @@ static int drbd_disconnected(int vnr, void *p, void *data)
  */
 static int drbd_send_features(struct drbd_tconn *tconn)
 {
-	/* ASSERT current == mdev->tconn->receiver ... */
-	struct p_connection_features *p = tconn->data.sbuf;
-	int err;
-
-	if (mutex_lock_interruptible(&tconn->data.mutex)) {
-		conn_err(tconn, "interrupted during initial handshake\n");
-		return -EINTR;
-	}
+	struct drbd_socket *sock;
+	struct p_connection_features *p;
 
-	if (tconn->data.socket == NULL) {
-		mutex_unlock(&tconn->data.mutex);
+	sock = &tconn->data;
+	p = conn_prepare_command(tconn, sock);
+	if (!p)
 		return -EIO;
-	}
-
 	memset(p, 0, sizeof(*p));
 	p->protocol_min = cpu_to_be32(PRO_VERSION_MIN);
 	p->protocol_max = cpu_to_be32(PRO_VERSION_MAX);
-	err = _conn_send_cmd(tconn, 0, &tconn->data, P_CONNECTION_FEATURES,
-			     &p->head, sizeof(*p), 0);
-	mutex_unlock(&tconn->data.mutex);
-	return err;
+	return conn_send_command(tconn, sock, P_CONNECTION_FEATURES, sizeof(*p), NULL, 0);
 }
 
 /*
@@ -4283,6 +4285,7 @@ static int drbd_do_auth(struct drbd_tconn *tconn)
 
 static int drbd_do_auth(struct drbd_tconn *tconn)
 {
+	struct drbd_socket *sock;
 	char my_challenge[CHALLENGE_LEN];  /* 64 Bytes... */
 	struct scatterlist sg;
 	char *response = NULL;
@@ -4294,6 +4297,8 @@ static int drbd_do_auth(struct drbd_tconn *tconn)
 	struct packet_info pi;
 	int err, rv;
 
+	/* FIXME: Put the challenge/response into the preallocated socket buffer.  */
+
 	desc.tfm = tconn->cram_hmac_tfm;
 	desc.flags = 0;
 
@@ -4307,7 +4312,14 @@ static int drbd_do_auth(struct drbd_tconn *tconn)
 
 	get_random_bytes(my_challenge, CHALLENGE_LEN);
 
-	rv = !conn_send_cmd2(tconn, P_AUTH_CHALLENGE, my_challenge, CHALLENGE_LEN);
+	sock = &tconn->data;
+	if (!conn_prepare_command(tconn, sock)) {
+		rv = 0;
+		goto fail;
+	}
+	rv = !conn_send_command(tconn, sock, P_AUTH_CHALLENGE,
+				sizeof(struct p_header),
+				my_challenge, CHALLENGE_LEN);
 	if (!rv)
 		goto fail;
 
@@ -4361,7 +4373,13 @@ static int drbd_do_auth(struct drbd_tconn *tconn)
 		goto fail;
 	}
 
-	rv = !conn_send_cmd2(tconn, P_AUTH_RESPONSE, response, resp_size);
+	if (!conn_prepare_command(tconn, sock)) {
+		rv = 0;
+		goto fail;
+	}
+	rv = !conn_send_command(tconn, sock, P_AUTH_RESPONSE,
+				sizeof(struct p_header),
+				response, resp_size);
 	if (!rv)
 		goto fail;
 

drivers/block/drbd/drbd_worker.c

@@ -1191,10 +1191,10 @@ int w_prev_work_done(struct drbd_work *w, int cancel)
 
 int w_send_barrier(struct drbd_work *w, int cancel)
 {
+	struct drbd_socket *sock;
 	struct drbd_tl_epoch *b = container_of(w, struct drbd_tl_epoch, w);
 	struct drbd_conf *mdev = w->mdev;
-	struct p_barrier *p = mdev->tconn->data.sbuf;
-	int err = 0;
+	struct p_barrier *p;
 
 	/* really avoid racing with tl_clear.  w.cb may have been referenced
 	 * just before it was reassigned and re-queued, so double check that.
@@ -1208,26 +1208,28 @@ int w_send_barrier(struct drbd_work *w, int cancel)
 	if (cancel)
 		return 0;
 
-	err = drbd_get_data_sock(mdev->tconn);
-	if (err)
-		return err;
+	sock = &mdev->tconn->data;
+	p = drbd_prepare_command(mdev, sock);
+	if (!p)
+		return -EIO;
 	p->barrier = b->br_number;
 	/* inc_ap_pending was done where this was queued.
 	 * dec_ap_pending will be done in got_BarrierAck
 	 * or (on connection loss) in w_clear_epoch.  */
-	err = _drbd_send_cmd(mdev, &mdev->tconn->data, P_BARRIER,
-			     &p->head, sizeof(*p), 0);
-	drbd_put_data_sock(mdev->tconn);
-
-	return err;
+	return drbd_send_command(mdev, sock, P_BARRIER, sizeof(*p), NULL, 0);
 }
 
 int w_send_write_hint(struct drbd_work *w, int cancel)
 {
 	struct drbd_conf *mdev = w->mdev;
+	struct drbd_socket *sock;
+
 	if (cancel)
 		return 0;
-	return drbd_send_short_cmd(mdev, P_UNPLUG_REMOTE);
+	sock = &mdev->tconn->data;
+	if (!drbd_prepare_command(mdev, sock))
+		return -EIO;
+	return drbd_send_command(mdev, sock, P_UNPLUG_REMOTE, sizeof(struct p_header), NULL, 0);
 }
 
 int w_send_out_of_sync(struct drbd_work *w, int cancel)