Commit 1e19ffc6 authored by Tejun Heo's avatar Tejun Heo

workqueue: implement per-cwq active work limit

Add cwq->nr_active, cwq->max_active and cwq->delayed_work.  nr_active
counts the number of active works per cwq.  A work is active if it's
flushable (colored) and is on cwq's worklist.  If nr_active reaches
max_active, new works are queued on cwq->delayed_work and activated
later as works on the cwq complete and decrement nr_active.

cwq->max_active can be specified via the new @max_active parameter to
__create_workqueue() and is set to 1 for all workqueues for now.  As
each cwq has only single worker now, this double queueing doesn't
cause any behavior difference visible to its users.

This will be used to reimplement freeze/thaw and implement shared
worker pool.
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
parent affee4b2
...@@ -225,11 +225,11 @@ enum { ...@@ -225,11 +225,11 @@ enum {
}; };
extern struct workqueue_struct * extern struct workqueue_struct *
__create_workqueue_key(const char *name, unsigned int flags, __create_workqueue_key(const char *name, unsigned int flags, int max_active,
struct lock_class_key *key, const char *lock_name); struct lock_class_key *key, const char *lock_name);
#ifdef CONFIG_LOCKDEP #ifdef CONFIG_LOCKDEP
#define __create_workqueue(name, flags) \ #define __create_workqueue(name, flags, max_active) \
({ \ ({ \
static struct lock_class_key __key; \ static struct lock_class_key __key; \
const char *__lock_name; \ const char *__lock_name; \
...@@ -239,20 +239,20 @@ __create_workqueue_key(const char *name, unsigned int flags, ...@@ -239,20 +239,20 @@ __create_workqueue_key(const char *name, unsigned int flags,
else \ else \
__lock_name = #name; \ __lock_name = #name; \
\ \
__create_workqueue_key((name), (flags), &__key, \ __create_workqueue_key((name), (flags), (max_active), \
__lock_name); \ &__key, __lock_name); \
}) })
#else #else
#define __create_workqueue(name, flags) \ #define __create_workqueue(name, flags, max_active) \
__create_workqueue_key((name), (flags), NULL, NULL) __create_workqueue_key((name), (flags), (max_active), NULL, NULL)
#endif #endif
#define create_workqueue(name) \ #define create_workqueue(name) \
__create_workqueue((name), 0) __create_workqueue((name), 0, 1)
#define create_freezeable_workqueue(name) \ #define create_freezeable_workqueue(name) \
__create_workqueue((name), WQ_FREEZEABLE | WQ_SINGLE_THREAD) __create_workqueue((name), WQ_FREEZEABLE | WQ_SINGLE_THREAD, 1)
#define create_singlethread_workqueue(name) \ #define create_singlethread_workqueue(name) \
__create_workqueue((name), WQ_SINGLE_THREAD) __create_workqueue((name), WQ_SINGLE_THREAD, 1)
extern void destroy_workqueue(struct workqueue_struct *wq); extern void destroy_workqueue(struct workqueue_struct *wq);
......
...@@ -77,6 +77,9 @@ struct cpu_workqueue_struct { ...@@ -77,6 +77,9 @@ struct cpu_workqueue_struct {
int flush_color; /* L: flushing color */ int flush_color; /* L: flushing color */
int nr_in_flight[WORK_NR_COLORS]; int nr_in_flight[WORK_NR_COLORS];
/* L: nr of in_flight works */ /* L: nr of in_flight works */
int nr_active; /* L: nr of active works */
int max_active; /* I: max active works */
struct list_head delayed_works; /* L: delayed works */
}; };
/* /*
...@@ -321,14 +324,24 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq, ...@@ -321,14 +324,24 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,
struct work_struct *work) struct work_struct *work)
{ {
struct cpu_workqueue_struct *cwq = target_cwq(cpu, wq); struct cpu_workqueue_struct *cwq = target_cwq(cpu, wq);
struct list_head *worklist;
unsigned long flags; unsigned long flags;
debug_work_activate(work); debug_work_activate(work);
spin_lock_irqsave(&cwq->lock, flags); spin_lock_irqsave(&cwq->lock, flags);
BUG_ON(!list_empty(&work->entry)); BUG_ON(!list_empty(&work->entry));
cwq->nr_in_flight[cwq->work_color]++; cwq->nr_in_flight[cwq->work_color]++;
insert_work(cwq, work, &cwq->worklist,
work_color_to_flags(cwq->work_color)); if (likely(cwq->nr_active < cwq->max_active)) {
cwq->nr_active++;
worklist = &cwq->worklist;
} else
worklist = &cwq->delayed_works;
insert_work(cwq, work, worklist, work_color_to_flags(cwq->work_color));
spin_unlock_irqrestore(&cwq->lock, flags); spin_unlock_irqrestore(&cwq->lock, flags);
} }
...@@ -584,6 +597,15 @@ static void move_linked_works(struct work_struct *work, struct list_head *head, ...@@ -584,6 +597,15 @@ static void move_linked_works(struct work_struct *work, struct list_head *head,
*nextp = n; *nextp = n;
} }
static void cwq_activate_first_delayed(struct cpu_workqueue_struct *cwq)
{
struct work_struct *work = list_first_entry(&cwq->delayed_works,
struct work_struct, entry);
move_linked_works(work, &cwq->worklist, NULL);
cwq->nr_active++;
}
/** /**
* cwq_dec_nr_in_flight - decrement cwq's nr_in_flight * cwq_dec_nr_in_flight - decrement cwq's nr_in_flight
* @cwq: cwq of interest * @cwq: cwq of interest
...@@ -602,6 +624,12 @@ static void cwq_dec_nr_in_flight(struct cpu_workqueue_struct *cwq, int color) ...@@ -602,6 +624,12 @@ static void cwq_dec_nr_in_flight(struct cpu_workqueue_struct *cwq, int color)
return; return;
cwq->nr_in_flight[color]--; cwq->nr_in_flight[color]--;
cwq->nr_active--;
/* one down, submit a delayed one */
if (!list_empty(&cwq->delayed_works) &&
cwq->nr_active < cwq->max_active)
cwq_activate_first_delayed(cwq);
/* is flush in progress and are we at the flushing tip? */ /* is flush in progress and are we at the flushing tip? */
if (likely(cwq->flush_color != color)) if (likely(cwq->flush_color != color))
...@@ -1505,6 +1533,7 @@ static void free_cwqs(struct cpu_workqueue_struct *cwqs) ...@@ -1505,6 +1533,7 @@ static void free_cwqs(struct cpu_workqueue_struct *cwqs)
struct workqueue_struct *__create_workqueue_key(const char *name, struct workqueue_struct *__create_workqueue_key(const char *name,
unsigned int flags, unsigned int flags,
int max_active,
struct lock_class_key *key, struct lock_class_key *key,
const char *lock_name) const char *lock_name)
{ {
...@@ -1513,6 +1542,8 @@ struct workqueue_struct *__create_workqueue_key(const char *name, ...@@ -1513,6 +1542,8 @@ struct workqueue_struct *__create_workqueue_key(const char *name,
bool failed = false; bool failed = false;
unsigned int cpu; unsigned int cpu;
max_active = clamp_val(max_active, 1, INT_MAX);
wq = kzalloc(sizeof(*wq), GFP_KERNEL); wq = kzalloc(sizeof(*wq), GFP_KERNEL);
if (!wq) if (!wq)
goto err; goto err;
...@@ -1544,8 +1575,10 @@ struct workqueue_struct *__create_workqueue_key(const char *name, ...@@ -1544,8 +1575,10 @@ struct workqueue_struct *__create_workqueue_key(const char *name,
cwq->cpu = cpu; cwq->cpu = cpu;
cwq->wq = wq; cwq->wq = wq;
cwq->flush_color = -1; cwq->flush_color = -1;
cwq->max_active = max_active;
spin_lock_init(&cwq->lock); spin_lock_init(&cwq->lock);
INIT_LIST_HEAD(&cwq->worklist); INIT_LIST_HEAD(&cwq->worklist);
INIT_LIST_HEAD(&cwq->delayed_works);
init_waitqueue_head(&cwq->more_work); init_waitqueue_head(&cwq->more_work);
if (failed) if (failed)
...@@ -1607,6 +1640,8 @@ void destroy_workqueue(struct workqueue_struct *wq) ...@@ -1607,6 +1640,8 @@ void destroy_workqueue(struct workqueue_struct *wq)
for (i = 0; i < WORK_NR_COLORS; i++) for (i = 0; i < WORK_NR_COLORS; i++)
BUG_ON(cwq->nr_in_flight[i]); BUG_ON(cwq->nr_in_flight[i]);
BUG_ON(cwq->nr_active);
BUG_ON(!list_empty(&cwq->delayed_works));
} }
free_cwqs(wq->cpu_wq); free_cwqs(wq->cpu_wq);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment