Commit 1050b27c authored by Ingo Molnar's avatar Ingo Molnar

sched/headers: Move cputime functionality from <linux/sched.h> and...

sched/headers: Move cputime functionality from <linux/sched.h> and <linux/cputime.h> into <linux/sched/cputime.h>

Move cputime related functionality out of <linux/sched.h>, as most code
that includes <linux/sched.h> does not use that functionality.

Move data types that are not included in task_struct directly to
the signal definitions, into <linux/sched/signal.h>.

Also merge the (small) existing <linux/cputime.h> header into <linux/sched/cputime.h>.
Acked-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 56cd6973
#ifndef __LINUX_CPUTIME_H
#define __LINUX_CPUTIME_H
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
#include <asm/cputime.h>
#ifndef cputime_to_nsecs
# define cputime_to_nsecs(__ct) \
(cputime_to_usecs(__ct) * NSEC_PER_USEC)
#endif
#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
#endif /* __LINUX_CPUTIME_H */
...@@ -219,14 +219,6 @@ struct prev_cputime { ...@@ -219,14 +219,6 @@ struct prev_cputime {
#endif #endif
}; };
static inline void prev_cputime_init(struct prev_cputime *prev)
{
#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
prev->utime = prev->stime = 0;
raw_spin_lock_init(&prev->lock);
#endif
}
/** /**
* struct task_cputime - collected CPU time counts * struct task_cputime - collected CPU time counts
* @utime: time spent in user mode, in nanoseconds * @utime: time spent in user mode, in nanoseconds
...@@ -248,40 +240,6 @@ struct task_cputime { ...@@ -248,40 +240,6 @@ struct task_cputime {
#define prof_exp stime #define prof_exp stime
#define sched_exp sum_exec_runtime #define sched_exp sum_exec_runtime
/*
* This is the atomic variant of task_cputime, which can be used for
* storing and updating task_cputime statistics without locking.
*/
struct task_cputime_atomic {
atomic64_t utime;
atomic64_t stime;
atomic64_t sum_exec_runtime;
};
#define INIT_CPUTIME_ATOMIC \
(struct task_cputime_atomic) { \
.utime = ATOMIC64_INIT(0), \
.stime = ATOMIC64_INIT(0), \
.sum_exec_runtime = ATOMIC64_INIT(0), \
}
/**
* struct thread_group_cputimer - thread group interval timer counts
* @cputime_atomic: atomic thread group interval timers.
* @running: true when there are timers running and
* @cputime_atomic receives updates.
* @checking_timer: true when a thread in the group is in the
* process of checking for thread group timers.
*
* This structure contains the version of task_cputime, above, that is
* used for thread group CPU timer calculations.
*/
struct thread_group_cputimer {
struct task_cputime_atomic cputime_atomic;
bool running;
bool checking_timer;
};
#include <linux/rwsem.h> #include <linux/rwsem.h>
#ifdef CONFIG_SCHED_INFO #ifdef CONFIG_SCHED_INFO
...@@ -1234,44 +1192,6 @@ static inline void put_task_struct(struct task_struct *t) ...@@ -1234,44 +1192,6 @@ static inline void put_task_struct(struct task_struct *t)
struct task_struct *task_rcu_dereference(struct task_struct **ptask); struct task_struct *task_rcu_dereference(struct task_struct **ptask);
struct task_struct *try_get_task_struct(struct task_struct **ptask); struct task_struct *try_get_task_struct(struct task_struct **ptask);
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
extern void task_cputime(struct task_struct *t,
u64 *utime, u64 *stime);
extern u64 task_gtime(struct task_struct *t);
#else
static inline void task_cputime(struct task_struct *t,
u64 *utime, u64 *stime)
{
*utime = t->utime;
*stime = t->stime;
}
static inline u64 task_gtime(struct task_struct *t)
{
return t->gtime;
}
#endif
#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
static inline void task_cputime_scaled(struct task_struct *t,
u64 *utimescaled,
u64 *stimescaled)
{
*utimescaled = t->utimescaled;
*stimescaled = t->stimescaled;
}
#else
static inline void task_cputime_scaled(struct task_struct *t,
u64 *utimescaled,
u64 *stimescaled)
{
task_cputime(t, utimescaled, stimescaled);
}
#endif
extern void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st);
extern void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st);
/* /*
* Per process flags * Per process flags
*/ */
...@@ -1395,9 +1315,6 @@ static inline int set_cpus_allowed_ptr(struct task_struct *p, ...@@ -1395,9 +1315,6 @@ static inline int set_cpus_allowed_ptr(struct task_struct *p,
#define cpu_relax_yield() cpu_relax() #define cpu_relax_yield() cpu_relax()
#endif #endif
extern unsigned long long
task_sched_runtime(struct task_struct *task);
/* sched_exec is called by processes performing an exec */ /* sched_exec is called by processes performing an exec */
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
extern void sched_exec(void); extern void sched_exec(void);
...@@ -1629,12 +1546,6 @@ static __always_inline bool need_resched(void) ...@@ -1629,12 +1546,6 @@ static __always_inline bool need_resched(void)
return unlikely(tif_need_resched()); return unlikely(tif_need_resched());
} }
/*
* Thread group CPU time accounting.
*/
void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times);
void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times);
/* /*
* Wrappers for p->thread_info->cpu access. No-op on UP. * Wrappers for p->thread_info->cpu access. No-op on UP.
*/ */
......
...@@ -2,6 +2,186 @@ ...@@ -2,6 +2,186 @@
#define _LINUX_SCHED_CPUTIME_H #define _LINUX_SCHED_CPUTIME_H
#include <linux/sched/signal.h> #include <linux/sched/signal.h>
#include <linux/cputime.h>
/*
* cputime accounting APIs:
*/
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
#include <asm/cputime.h>
#ifndef cputime_to_nsecs
# define cputime_to_nsecs(__ct) \
(cputime_to_usecs(__ct) * NSEC_PER_USEC)
#endif
#endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
extern void task_cputime(struct task_struct *t,
u64 *utime, u64 *stime);
extern u64 task_gtime(struct task_struct *t);
#else
static inline void task_cputime(struct task_struct *t,
u64 *utime, u64 *stime)
{
*utime = t->utime;
*stime = t->stime;
}
static inline u64 task_gtime(struct task_struct *t)
{
return t->gtime;
}
#endif
#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
static inline void task_cputime_scaled(struct task_struct *t,
u64 *utimescaled,
u64 *stimescaled)
{
*utimescaled = t->utimescaled;
*stimescaled = t->stimescaled;
}
#else
static inline void task_cputime_scaled(struct task_struct *t,
u64 *utimescaled,
u64 *stimescaled)
{
task_cputime(t, utimescaled, stimescaled);
}
#endif
extern void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st);
extern void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st);
/*
* Thread group CPU time accounting.
*/
void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times);
void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times);
/*
* The following are functions that support scheduler-internal time accounting.
* These functions are generally called at the timer tick. None of this depends
* on CONFIG_SCHEDSTATS.
*/
/**
* get_running_cputimer - return &tsk->signal->cputimer if cputimer is running
*
* @tsk: Pointer to target task.
*/
#ifdef CONFIG_POSIX_TIMERS
static inline
struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk)
{
struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;
/* Check if cputimer isn't running. This is accessed without locking. */
if (!READ_ONCE(cputimer->running))
return NULL;
/*
* After we flush the task's sum_exec_runtime to sig->sum_sched_runtime
* in __exit_signal(), we won't account to the signal struct further
* cputime consumed by that task, even though the task can still be
* ticking after __exit_signal().
*
* In order to keep a consistent behaviour between thread group cputime
* and thread group cputimer accounting, lets also ignore the cputime
* elapsing after __exit_signal() in any thread group timer running.
*
* This makes sure that POSIX CPU clocks and timers are synchronized, so
* that a POSIX CPU timer won't expire while the corresponding POSIX CPU
* clock delta is behind the expiring timer value.
*/
if (unlikely(!tsk->sighand))
return NULL;
return cputimer;
}
#else
static inline
struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk)
{
return NULL;
}
#endif
/**
* account_group_user_time - Maintain utime for a thread group.
*
* @tsk: Pointer to task structure.
* @cputime: Time value by which to increment the utime field of the
* thread_group_cputime structure.
*
* If thread group time is being maintained, get the structure for the
* running CPU and update the utime field there.
*/
static inline void account_group_user_time(struct task_struct *tsk,
u64 cputime)
{
struct thread_group_cputimer *cputimer = get_running_cputimer(tsk);
if (!cputimer)
return;
atomic64_add(cputime, &cputimer->cputime_atomic.utime);
}
/**
* account_group_system_time - Maintain stime for a thread group.
*
* @tsk: Pointer to task structure.
* @cputime: Time value by which to increment the stime field of the
* thread_group_cputime structure.
*
* If thread group time is being maintained, get the structure for the
* running CPU and update the stime field there.
*/
static inline void account_group_system_time(struct task_struct *tsk,
u64 cputime)
{
struct thread_group_cputimer *cputimer = get_running_cputimer(tsk);
if (!cputimer)
return;
atomic64_add(cputime, &cputimer->cputime_atomic.stime);
}
/**
* account_group_exec_runtime - Maintain exec runtime for a thread group.
*
* @tsk: Pointer to task structure.
* @ns: Time value by which to increment the sum_exec_runtime field
* of the thread_group_cputime structure.
*
* If thread group time is being maintained, get the structure for the
* running CPU and update the sum_exec_runtime field there.
*/
static inline void account_group_exec_runtime(struct task_struct *tsk,
unsigned long long ns)
{
struct thread_group_cputimer *cputimer = get_running_cputimer(tsk);
if (!cputimer)
return;
atomic64_add(ns, &cputimer->cputime_atomic.sum_exec_runtime);
}
static inline void prev_cputime_init(struct prev_cputime *prev)
{
#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
prev->utime = prev->stime = 0;
raw_spin_lock_init(&prev->lock);
#endif
}
extern unsigned long long
task_sched_runtime(struct task_struct *task);
#endif /* _LINUX_SCHED_CPUTIME_H */ #endif /* _LINUX_SCHED_CPUTIME_H */
...@@ -35,6 +35,39 @@ struct cpu_itimer { ...@@ -35,6 +35,39 @@ struct cpu_itimer {
u64 incr; u64 incr;
}; };
/*
* This is the atomic variant of task_cputime, which can be used for
* storing and updating task_cputime statistics without locking.
*/
struct task_cputime_atomic {
atomic64_t utime;
atomic64_t stime;
atomic64_t sum_exec_runtime;
};
#define INIT_CPUTIME_ATOMIC \
(struct task_cputime_atomic) { \
.utime = ATOMIC64_INIT(0), \
.stime = ATOMIC64_INIT(0), \
.sum_exec_runtime = ATOMIC64_INIT(0), \
}
/**
* struct thread_group_cputimer - thread group interval timer counts
* @cputime_atomic: atomic thread group interval timers.
* @running: true when there are timers running and
* @cputime_atomic receives updates.
* @checking_timer: true when a thread in the group is in the
* process of checking for thread group timers.
*
* This structure contains the version of task_cputime, above, that is
* used for thread group CPU timer calculations.
*/
struct thread_group_cputimer {
struct task_cputime_atomic cputime_atomic;
bool running;
bool checking_timer;
};
/* /*
* NOTE! "signal_struct" does not have its own * NOTE! "signal_struct" does not have its own
* locking, because a shared signal_struct always * locking, because a shared signal_struct always
......
...@@ -164,114 +164,3 @@ sched_info_switch(struct rq *rq, ...@@ -164,114 +164,3 @@ sched_info_switch(struct rq *rq,
#define sched_info_arrive(rq, next) do { } while (0) #define sched_info_arrive(rq, next) do { } while (0)
#define sched_info_switch(rq, t, next) do { } while (0) #define sched_info_switch(rq, t, next) do { } while (0)
#endif /* CONFIG_SCHED_INFO */ #endif /* CONFIG_SCHED_INFO */
/*
* The following are functions that support scheduler-internal time accounting.
* These functions are generally called at the timer tick. None of this depends
* on CONFIG_SCHEDSTATS.
*/
/**
* get_running_cputimer - return &tsk->signal->cputimer if cputimer is running
*
* @tsk: Pointer to target task.
*/
#ifdef CONFIG_POSIX_TIMERS
static inline
struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk)
{
struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;
/* Check if cputimer isn't running. This is accessed without locking. */
if (!READ_ONCE(cputimer->running))
return NULL;
/*
* After we flush the task's sum_exec_runtime to sig->sum_sched_runtime
* in __exit_signal(), we won't account to the signal struct further
* cputime consumed by that task, even though the task can still be
* ticking after __exit_signal().
*
* In order to keep a consistent behaviour between thread group cputime
* and thread group cputimer accounting, lets also ignore the cputime
* elapsing after __exit_signal() in any thread group timer running.
*
* This makes sure that POSIX CPU clocks and timers are synchronized, so
* that a POSIX CPU timer won't expire while the corresponding POSIX CPU
* clock delta is behind the expiring timer value.
*/
if (unlikely(!tsk->sighand))
return NULL;
return cputimer;
}
#else
static inline
struct thread_group_cputimer *get_running_cputimer(struct task_struct *tsk)
{
return NULL;
}
#endif
/**
* account_group_user_time - Maintain utime for a thread group.
*
* @tsk: Pointer to task structure.
* @cputime: Time value by which to increment the utime field of the
* thread_group_cputime structure.
*
* If thread group time is being maintained, get the structure for the
* running CPU and update the utime field there.
*/
static inline void account_group_user_time(struct task_struct *tsk,
u64 cputime)
{
struct thread_group_cputimer *cputimer = get_running_cputimer(tsk);
if (!cputimer)
return;
atomic64_add(cputime, &cputimer->cputime_atomic.utime);
}
/**
* account_group_system_time - Maintain stime for a thread group.
*
* @tsk: Pointer to task structure.
* @cputime: Time value by which to increment the stime field of the
* thread_group_cputime structure.
*
* If thread group time is being maintained, get the structure for the
* running CPU and update the stime field there.
*/
static inline void account_group_system_time(struct task_struct *tsk,
u64 cputime)
{
struct thread_group_cputimer *cputimer = get_running_cputimer(tsk);
if (!cputimer)
return;
atomic64_add(cputime, &cputimer->cputime_atomic.stime);
}
/**
* account_group_exec_runtime - Maintain exec runtime for a thread group.
*
* @tsk: Pointer to task structure.
* @ns: Time value by which to increment the sum_exec_runtime field
* of the thread_group_cputime structure.
*
* If thread group time is being maintained, get the structure for the
* running CPU and update the sum_exec_runtime field there.
*/
static inline void account_group_exec_runtime(struct task_struct *tsk,
unsigned long long ns)
{
struct thread_group_cputimer *cputimer = get_running_cputimer(tsk);
if (!cputimer)
return;
atomic64_add(ns, &cputimer->cputime_atomic.sum_exec_runtime);
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment