Commit e3658538 authored by Davidlohr Bueso's avatar Davidlohr Bueso Committed by Linus Torvalds

ipc/msg: batch queue sender wakeups

Currently the use of wake_qs in sysv msg queues are only for the receiver
tasks that are blocked on the queue.  But blocked sender tasks (due to
queue size constraints) still are awoken with the ipc object lock held,
which can be a problem particularly for small sized queues and far from
gracious for -rt (just like it was for the receiver side).

The paths that actually wakeup a sender are obviously related to when we
are either getting rid of the queue or after (some) space is freed-up
after a receiver takes the msg (msgrcv).  Furthermore, with the exception
of msgrcv, we can always piggy-back on expunge_all that has its own tasks
lined-up for waking.  Finally, upon unlinking the message, it should be no
problem delaying the wakeups a bit until after we've released the lock.

Link: http://lkml.kernel.org/r/1469748819-19484-3-git-send-email-dave@stgolabs.netSigned-off-by: default avatarDavidlohr Bueso <dbueso@suse.de>
Acked-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Manfred Spraul <manfred@colorfullife.com>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent ee51636c
...@@ -166,14 +166,15 @@ static inline void ss_del(struct msg_sender *mss) ...@@ -166,14 +166,15 @@ static inline void ss_del(struct msg_sender *mss)
list_del(&mss->list); list_del(&mss->list);
} }
static void ss_wakeup(struct list_head *h, int kill) static void ss_wakeup(struct list_head *h,
struct wake_q_head *wake_q, int kill)
{ {
struct msg_sender *mss, *t; struct msg_sender *mss, *t;
list_for_each_entry_safe(mss, t, h, list) { list_for_each_entry_safe(mss, t, h, list) {
if (kill) if (kill)
mss->list.next = NULL; mss->list.next = NULL;
wake_up_process(mss->tsk); wake_q_add(wake_q, mss->tsk);
} }
} }
...@@ -203,7 +204,7 @@ static void freeque(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp) ...@@ -203,7 +204,7 @@ static void freeque(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
WAKE_Q(wake_q); WAKE_Q(wake_q);
expunge_all(msq, -EIDRM, &wake_q); expunge_all(msq, -EIDRM, &wake_q);
ss_wakeup(&msq->q_senders, 1); ss_wakeup(&msq->q_senders, &wake_q, 1);
msg_rmid(ns, msq); msg_rmid(ns, msq);
ipc_unlock_object(&msq->q_perm); ipc_unlock_object(&msq->q_perm);
wake_up_q(&wake_q); wake_up_q(&wake_q);
...@@ -331,7 +332,6 @@ static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd, ...@@ -331,7 +332,6 @@ static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd,
struct kern_ipc_perm *ipcp; struct kern_ipc_perm *ipcp;
struct msqid64_ds uninitialized_var(msqid64); struct msqid64_ds uninitialized_var(msqid64);
struct msg_queue *msq; struct msg_queue *msq;
WAKE_Q(wake_q);
int err; int err;
if (cmd == IPC_SET) { if (cmd == IPC_SET) {
...@@ -362,6 +362,9 @@ static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd, ...@@ -362,6 +362,9 @@ static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd,
freeque(ns, ipcp); freeque(ns, ipcp);
goto out_up; goto out_up;
case IPC_SET: case IPC_SET:
{
WAKE_Q(wake_q);
if (msqid64.msg_qbytes > ns->msg_ctlmnb && if (msqid64.msg_qbytes > ns->msg_ctlmnb &&
!capable(CAP_SYS_RESOURCE)) { !capable(CAP_SYS_RESOURCE)) {
err = -EPERM; err = -EPERM;
...@@ -376,15 +379,21 @@ static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd, ...@@ -376,15 +379,21 @@ static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd,
msq->q_qbytes = msqid64.msg_qbytes; msq->q_qbytes = msqid64.msg_qbytes;
msq->q_ctime = get_seconds(); msq->q_ctime = get_seconds();
/* sleeping receivers might be excluded by /*
* Sleeping receivers might be excluded by
* stricter permissions. * stricter permissions.
*/ */
expunge_all(msq, -EAGAIN, &wake_q); expunge_all(msq, -EAGAIN, &wake_q);
/* sleeping senders might be able to send /*
* Sleeping senders might be able to send
* due to a larger queue size. * due to a larger queue size.
*/ */
ss_wakeup(&msq->q_senders, 0); ss_wakeup(&msq->q_senders, &wake_q, 0);
break; ipc_unlock_object(&msq->q_perm);
wake_up_q(&wake_q);
goto out_unlock1;
}
default: default:
err = -EINVAL; err = -EINVAL;
goto out_unlock1; goto out_unlock1;
...@@ -392,7 +401,6 @@ static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd, ...@@ -392,7 +401,6 @@ static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd,
out_unlock0: out_unlock0:
ipc_unlock_object(&msq->q_perm); ipc_unlock_object(&msq->q_perm);
wake_up_q(&wake_q);
out_unlock1: out_unlock1:
rcu_read_unlock(); rcu_read_unlock();
out_up: out_up:
...@@ -809,6 +817,7 @@ long do_msgrcv(int msqid, void __user *buf, size_t bufsz, long msgtyp, int msgfl ...@@ -809,6 +817,7 @@ long do_msgrcv(int msqid, void __user *buf, size_t bufsz, long msgtyp, int msgfl
struct msg_queue *msq; struct msg_queue *msq;
struct ipc_namespace *ns; struct ipc_namespace *ns;
struct msg_msg *msg, *copy = NULL; struct msg_msg *msg, *copy = NULL;
WAKE_Q(wake_q);
ns = current->nsproxy->ipc_ns; ns = current->nsproxy->ipc_ns;
...@@ -873,7 +882,7 @@ long do_msgrcv(int msqid, void __user *buf, size_t bufsz, long msgtyp, int msgfl ...@@ -873,7 +882,7 @@ long do_msgrcv(int msqid, void __user *buf, size_t bufsz, long msgtyp, int msgfl
msq->q_cbytes -= msg->m_ts; msq->q_cbytes -= msg->m_ts;
atomic_sub(msg->m_ts, &ns->msg_bytes); atomic_sub(msg->m_ts, &ns->msg_bytes);
atomic_dec(&ns->msg_hdrs); atomic_dec(&ns->msg_hdrs);
ss_wakeup(&msq->q_senders, 0); ss_wakeup(&msq->q_senders, &wake_q, 0);
goto out_unlock0; goto out_unlock0;
} }
...@@ -945,6 +954,7 @@ long do_msgrcv(int msqid, void __user *buf, size_t bufsz, long msgtyp, int msgfl ...@@ -945,6 +954,7 @@ long do_msgrcv(int msqid, void __user *buf, size_t bufsz, long msgtyp, int msgfl
out_unlock0: out_unlock0:
ipc_unlock_object(&msq->q_perm); ipc_unlock_object(&msq->q_perm);
wake_up_q(&wake_q);
out_unlock1: out_unlock1:
rcu_read_unlock(); rcu_read_unlock();
if (IS_ERR(msg)) { if (IS_ERR(msg)) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment