Commit 6baf2730 authored by Kent Overstreet's avatar Kent Overstreet Committed by Kent Overstreet

bcachefs: Inline fast path of bch2_increment_clock()

Shaving more cycles.
Signed-off-by: default avatarKent Overstreet <kent.overstreet@gmail.com>
Signed-off-by: default avatarKent Overstreet <kent.overstreet@linux.dev>
parent f58c22e7
...@@ -135,17 +135,16 @@ static struct io_timer *get_expired_timer(struct io_clock *clock, ...@@ -135,17 +135,16 @@ static struct io_timer *get_expired_timer(struct io_clock *clock,
return ret; return ret;
} }
void bch2_increment_clock(struct bch_fs *c, unsigned sectors, int rw) void __bch2_increment_clock(struct io_clock *clock)
{ {
struct io_clock *clock = &c->io_clock[rw];
struct io_timer *timer; struct io_timer *timer;
unsigned long now; unsigned long now;
unsigned sectors;
/* Buffer up one megabyte worth of IO in the percpu counter */ /* Buffer up one megabyte worth of IO in the percpu counter */
preempt_disable(); preempt_disable();
if (likely(this_cpu_add_return(*clock->pcpu_buf, sectors) < if (this_cpu_read(*clock->pcpu_buf) < IO_CLOCK_PCPU_SECTORS) {
IO_CLOCK_PCPU_SECTORS)) {
preempt_enable(); preempt_enable();
return; return;
} }
......
...@@ -6,7 +6,18 @@ void bch2_io_timer_add(struct io_clock *, struct io_timer *); ...@@ -6,7 +6,18 @@ void bch2_io_timer_add(struct io_clock *, struct io_timer *);
void bch2_io_timer_del(struct io_clock *, struct io_timer *); void bch2_io_timer_del(struct io_clock *, struct io_timer *);
void bch2_kthread_io_clock_wait(struct io_clock *, unsigned long, void bch2_kthread_io_clock_wait(struct io_clock *, unsigned long,
unsigned long); unsigned long);
void bch2_increment_clock(struct bch_fs *, unsigned, int);
void __bch2_increment_clock(struct io_clock *);
static inline void bch2_increment_clock(struct bch_fs *c, unsigned sectors,
int rw)
{
struct io_clock *clock = &c->io_clock[rw];
if (unlikely(this_cpu_add_return(*clock->pcpu_buf, sectors) >=
IO_CLOCK_PCPU_SECTORS))
__bch2_increment_clock(clock);
}
void bch2_io_clock_schedule_timeout(struct io_clock *, unsigned long); void bch2_io_clock_schedule_timeout(struct io_clock *, unsigned long);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment