Skip to content

L
linux
Commit bceb7acc authored by Linus Torvalds's avatar Linus Torvalds
Browse files
Options

Merge tag 'perf-core-2023-10-28' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip 

Pull performance event updates from Ingo Molnar:
 - Add AMD Unified Memory Controller (UMC) events introduced with Zen 4
 - Simplify & clean up the uncore management code
 - Fall back from RDPMC to RDMSR on certain uncore PMUs
 - Improve per-package and cstate event reading
 - Extend the Intel ref-cycles event to GP counters
 - Fix Intel MTL event constraints
 - Improve the Intel hybrid CPU handling code
 - Micro-optimize the RAPL code
 - Optimize perf_cgroup_switch()
 - Improve large AUX area error handling
 - Misc fixes and cleanups

* tag 'perf-core-2023-10-28' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (26 commits)
  perf/x86/amd/uncore: Pass through error code for initialization failures, instead of -ENODEV
  perf/x86/amd/uncore: Fix uninitialized return value in amd_uncore_init()
  x86/cpu: Fix the AMD Fam 17h, Fam 19h, Zen2 and Zen4 MSR enumerations
  perf: Optimize perf_cgroup_switch()
  perf/x86/amd/uncore: Add memory controller support
  perf/x86/amd/uncore: Add group exclusivity
  perf/x86/amd/uncore: Use rdmsr if rdpmc is unavailable
  perf/x86/amd/uncore: Move discovery and registration
  perf/x86/amd/uncore: Refactor uncore management
  perf/core: Allow reading package events from perf_event_read_local
  perf/x86/cstate: Allow reading the package statistics from local CPU
  perf/x86/intel/pt: Fix kernel-doc comments
  perf/x86/rapl: Annotate 'struct rapl_pmus' with __counted_by
  perf/core: Rename perf_proc_update_handler() -> perf_event_max_sample_rate_handler(), for readability
  perf/x86/rapl: Fix "Using plain integer as NULL pointer" Sparse warning
  perf/x86/rapl: Use local64_try_cmpxchg in rapl_event_update()
  perf/x86/rapl: Stop doing cpu_relax() in the local64_cmpxchg() loop in rapl_event_update()
  perf/core: Bail out early if the request AUX area is out of bound
  perf/x86/intel: Extend the ref-cycles event to GP counters
  perf/x86/intel: Fix broken fixed event constraints extension
  ...
parents cd063c8b 744940f1
Show whitespace changes
Inline Side-by-side
Showing with 1093 additions and 689 deletions
arch/x86/events/amd/uncore.c
View file @ bceb7acc
...@@ -26,57 +26,66 @@ ...@@ -26,57 +26,66 @@
#define RDPMC_BASE_LLC 10 #define RDPMC_BASE_LLC 10
#define COUNTER_SHIFT 16 #define COUNTER_SHIFT 16
#define UNCORE_NAME_LEN 16
#define UNCORE_GROUP_MAX 256
#undef pr_fmt #undef pr_fmt
#define pr_fmt(fmt) "amd_uncore: " fmt #define pr_fmt(fmt) "amd_uncore: " fmt
static int pmu_version; static int pmu_version;
static int num_counters_llc;
static int num_counters_nb;
static bool l3_mask;
static HLIST_HEAD(uncore_unused_list); struct amd_uncore_ctx {
struct amd_uncore {
int id;
int refcnt; int refcnt;
int cpu; int cpu;
struct perf_event **events;
struct hlist_node node;
};
struct amd_uncore_pmu {
char name[UNCORE_NAME_LEN];
int num_counters; int num_counters;
int rdpmc_base; int rdpmc_base;
u32 msr_base; u32 msr_base;
cpumask_t *active_mask; int group;
struct pmu *pmu; cpumask_t active_mask;
struct perf_event **events; struct pmu pmu;
struct hlist_node node; struct amd_uncore_ctx * __percpu *ctx;
}; };
static struct amd_uncore * __percpu *amd_uncore_nb; enum {
static struct amd_uncore * __percpu *amd_uncore_llc; UNCORE_TYPE_DF,
UNCORE_TYPE_L3,
UNCORE_TYPE_UMC,
static struct pmu amd_nb_pmu; UNCORE_TYPE_MAX
static struct pmu amd_llc_pmu; };
static cpumask_t amd_nb_active_mask; union amd_uncore_info {
static cpumask_t amd_llc_active_mask; struct {
u64 aux_data:32; /* auxiliary data */
u64 num_pmcs:8; /* number of counters */
u64 gid:8; /* group id */
u64 cid:8; /* context id */
} split;
u64 full;
};
static bool is_nb_event(struct perf_event *event) struct amd_uncore {
{ union amd_uncore_info * __percpu info;
return event->pmu->type == amd_nb_pmu.type; struct amd_uncore_pmu *pmus;
} unsigned int num_pmus;
bool init_done;
void (*scan)(struct amd_uncore *uncore, unsigned int cpu);
int (*init)(struct amd_uncore *uncore, unsigned int cpu);
void (*move)(struct amd_uncore *uncore, unsigned int cpu);
void (*free)(struct amd_uncore *uncore, unsigned int cpu);
};
static bool is_llc_event(struct perf_event *event) static struct amd_uncore uncores[UNCORE_TYPE_MAX];
{
return event->pmu->type == amd_llc_pmu.type;
}
static struct amd_uncore *event_to_amd_uncore(struct perf_event *event) static struct amd_uncore_pmu *event_to_amd_uncore_pmu(struct perf_event *event)
{ {
if (is_nb_event(event) && amd_uncore_nb) return container_of(event->pmu, struct amd_uncore_pmu, pmu);
return *per_cpu_ptr(amd_uncore_nb, event->cpu);
else if (is_llc_event(event) && amd_uncore_llc)
return *per_cpu_ptr(amd_uncore_llc, event->cpu);
return NULL;
} }
static void amd_uncore_read(struct perf_event *event) static void amd_uncore_read(struct perf_event *event)
...@@ -91,7 +100,16 @@ static void amd_uncore_read(struct perf_event *event) ...@@ -91,7 +100,16 @@ static void amd_uncore_read(struct perf_event *event)
*/ */
prev = local64_read(&hwc->prev_count); prev = local64_read(&hwc->prev_count);
/*
* Some uncore PMUs do not have RDPMC assignments. In such cases,
* read counts directly from the corresponding PERF_CTR.
*/
if (hwc->event_base_rdpmc < 0)
rdmsrl(hwc->event_base, new);
else
rdpmcl(hwc->event_base_rdpmc, new); rdpmcl(hwc->event_base_rdpmc, new);
local64_set(&hwc->prev_count, new); local64_set(&hwc->prev_count, new);
delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT); delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
delta >>= COUNTER_SHIFT; delta >>= COUNTER_SHIFT;
...@@ -118,7 +136,7 @@ static void amd_uncore_stop(struct perf_event *event, int flags) ...@@ -118,7 +136,7 @@ static void amd_uncore_stop(struct perf_event *event, int flags)
hwc->state |= PERF_HES_STOPPED; hwc->state |= PERF_HES_STOPPED;
if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
amd_uncore_read(event); event->pmu->read(event);
hwc->state |= PERF_HES_UPTODATE; hwc->state |= PERF_HES_UPTODATE;
} }
} }
...@@ -126,15 +144,16 @@ static void amd_uncore_stop(struct perf_event *event, int flags) ...@@ -126,15 +144,16 @@ static void amd_uncore_stop(struct perf_event *event, int flags)
static int amd_uncore_add(struct perf_event *event, int flags) static int amd_uncore_add(struct perf_event *event, int flags)
{ {
int i; int i;
struct amd_uncore *uncore = event_to_amd_uncore(event); struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
struct hw_perf_event *hwc = &event->hw; struct hw_perf_event *hwc = &event->hw;
/* are we already assigned? */ /* are we already assigned? */
if (hwc->idx != -1 && uncore->events[hwc->idx] == event) if (hwc->idx != -1 && ctx->events[hwc->idx] == event)
goto out; goto out;
for (i = 0; i < uncore->num_counters; i++) { for (i = 0; i < pmu->num_counters; i++) {
if (uncore->events[i] == event) { if (ctx->events[i] == event) {
hwc->idx = i; hwc->idx = i;
goto out; goto out;
} }
...@@ -142,8 +161,8 @@ static int amd_uncore_add(struct perf_event *event, int flags) ...@@ -142,8 +161,8 @@ static int amd_uncore_add(struct perf_event *event, int flags)
/* if not, take the first available counter */ /* if not, take the first available counter */
hwc->idx = -1; hwc->idx = -1;
for (i = 0; i < uncore->num_counters; i++) { for (i = 0; i < pmu->num_counters; i++) {
if (cmpxchg(&uncore->events[i], NULL, event) == NULL) { if (cmpxchg(&ctx->events[i], NULL, event) == NULL) {
hwc->idx = i; hwc->idx = i;
break; break;
} }
...@@ -153,23 +172,16 @@ static int amd_uncore_add(struct perf_event *event, int flags) ...@@ -153,23 +172,16 @@ static int amd_uncore_add(struct perf_event *event, int flags)
if (hwc->idx == -1) if (hwc->idx == -1)
return -EBUSY; return -EBUSY;
hwc->config_base = uncore->msr_base + (2 * hwc->idx); hwc->config_base = pmu->msr_base + (2 * hwc->idx);
hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx); hwc->event_base = pmu->msr_base + 1 + (2 * hwc->idx);
hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx; hwc->event_base_rdpmc = pmu->rdpmc_base + hwc->idx;
hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
/* if (pmu->rdpmc_base < 0)
* The first four DF counters are accessible via RDPMC index 6 to 9 hwc->event_base_rdpmc = -1;
* followed by the L3 counters from index 10 to 15. For processors
* with more than four DF counters, the DF RDPMC assignments become
* discontiguous as the additional counters are accessible starting
* from index 16.
*/
if (is_nb_event(event) && hwc->idx >= NUM_COUNTERS_NB)
hwc->event_base_rdpmc += NUM_COUNTERS_L3;
if (flags & PERF_EF_START) if (flags & PERF_EF_START)
amd_uncore_start(event, PERF_EF_RELOAD); event->pmu->start(event, PERF_EF_RELOAD);
return 0; return 0;
} }
...@@ -177,55 +189,36 @@ static int amd_uncore_add(struct perf_event *event, int flags) ...@@ -177,55 +189,36 @@ static int amd_uncore_add(struct perf_event *event, int flags)
static void amd_uncore_del(struct perf_event *event, int flags) static void amd_uncore_del(struct perf_event *event, int flags)
{ {
int i; int i;
struct amd_uncore *uncore = event_to_amd_uncore(event); struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event);
struct amd_uncore_ctx *ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
struct hw_perf_event *hwc = &event->hw; struct hw_perf_event *hwc = &event->hw;
amd_uncore_stop(event, PERF_EF_UPDATE); event->pmu->stop(event, PERF_EF_UPDATE);
for (i = 0; i < uncore->num_counters; i++) { for (i = 0; i < pmu->num_counters; i++) {
if (cmpxchg(&uncore->events[i], event, NULL) == event) if (cmpxchg(&ctx->events[i], event, NULL) == event)
break; break;
} }
hwc->idx = -1; hwc->idx = -1;
} }
/*
* Return a full thread and slice mask unless user
* has provided them
*/
static u64 l3_thread_slice_mask(u64 config)
{
if (boot_cpu_data.x86 <= 0x18)
return ((config & AMD64_L3_SLICE_MASK) ? : AMD64_L3_SLICE_MASK) |
((config & AMD64_L3_THREAD_MASK) ? : AMD64_L3_THREAD_MASK);
/*
* If the user doesn't specify a threadmask, they're not trying to
* count core 0, so we enable all cores & threads.
* We'll also assume that they want to count slice 0 if they specify
* a threadmask and leave sliceid and enallslices unpopulated.
*/
if (!(config & AMD64_L3_F19H_THREAD_MASK))
return AMD64_L3_F19H_THREAD_MASK | AMD64_L3_EN_ALL_SLICES |
AMD64_L3_EN_ALL_CORES;
return config & (AMD64_L3_F19H_THREAD_MASK | AMD64_L3_SLICEID_MASK |
AMD64_L3_EN_ALL_CORES | AMD64_L3_EN_ALL_SLICES |
AMD64_L3_COREID_MASK);
}
static int amd_uncore_event_init(struct perf_event *event) static int amd_uncore_event_init(struct perf_event *event)
{ {
struct amd_uncore *uncore; struct amd_uncore_pmu *pmu;
struct amd_uncore_ctx *ctx;
struct hw_perf_event *hwc = &event->hw; struct hw_perf_event *hwc = &event->hw;
u64 event_mask = AMD64_RAW_EVENT_MASK_NB;
if (event->attr.type != event->pmu->type) if (event->attr.type != event->pmu->type)
return -ENOENT; return -ENOENT;
if (pmu_version >= 2 && is_nb_event(event)) if (event->cpu < 0)
event_mask = AMD64_PERFMON_V2_RAW_EVENT_MASK_NB; return -EINVAL;
pmu = event_to_amd_uncore_pmu(event);
ctx = *per_cpu_ptr(pmu->ctx, event->cpu);
if (!ctx)
return -ENODEV;
/* /*
* NB and Last level cache counters (MSRs) are shared across all cores * NB and Last level cache counters (MSRs) are shared across all cores
...@@ -235,28 +228,14 @@ static int amd_uncore_event_init(struct perf_event *event) ...@@ -235,28 +228,14 @@ static int amd_uncore_event_init(struct perf_event *event)
* out. So we do not support sampling and per-thread events via * out. So we do not support sampling and per-thread events via
* CAP_NO_INTERRUPT, and we do not enable counter overflow interrupts: * CAP_NO_INTERRUPT, and we do not enable counter overflow interrupts:
*/ */
hwc->config = event->attr.config & event_mask; hwc->config = event->attr.config;
hwc->idx = -1; hwc->idx = -1;
if (event->cpu < 0)
return -EINVAL;
/*
* SliceMask and ThreadMask need to be set for certain L3 events.
* For other events, the two fields do not affect the count.
*/
if (l3_mask && is_llc_event(event))
hwc->config |= l3_thread_slice_mask(event->attr.config);
uncore = event_to_amd_uncore(event);
if (!uncore)
return -ENODEV;
/* /*
* since request can come in to any of the shared cores, we will remap * since request can come in to any of the shared cores, we will remap
* to a single common cpu. * to a single common cpu.
*/ */
event->cpu = uncore->cpu; event->cpu = ctx->cpu;
return 0; return 0;
} }
...@@ -278,17 +257,10 @@ static ssize_t amd_uncore_attr_show_cpumask(struct device *dev, ...@@ -278,17 +257,10 @@ static ssize_t amd_uncore_attr_show_cpumask(struct device *dev,
struct device_attribute *attr, struct device_attribute *attr,
char *buf) char *buf)
{ {
cpumask_t *active_mask; struct pmu *ptr = dev_get_drvdata(dev);
struct pmu *pmu = dev_get_drvdata(dev); struct amd_uncore_pmu *pmu = container_of(ptr, struct amd_uncore_pmu, pmu);
if (pmu->type == amd_nb_pmu.type)
active_mask = &amd_nb_active_mask;
else if (pmu->type == amd_llc_pmu.type)
active_mask = &amd_llc_active_mask;
else
return 0;
return cpumap_print_to_pagebuf(true, buf, active_mask); return cpumap_print_to_pagebuf(true, buf, &pmu->active_mask);
} }
static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL); static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL);
...@@ -315,7 +287,7 @@ static struct device_attribute format_attr_##_var = \ ...@@ -315,7 +287,7 @@ static struct device_attribute format_attr_##_var = \
DEFINE_UNCORE_FORMAT_ATTR(event12, event, "config:0-7,32-35"); DEFINE_UNCORE_FORMAT_ATTR(event12, event, "config:0-7,32-35");
DEFINE_UNCORE_FORMAT_ATTR(event14, event, "config:0-7,32-35,59-60"); /* F17h+ DF */ DEFINE_UNCORE_FORMAT_ATTR(event14, event, "config:0-7,32-35,59-60"); /* F17h+ DF */
DEFINE_UNCORE_FORMAT_ATTR(event14v2, event, "config:0-7,32-37"); /* PerfMonV2 DF */ DEFINE_UNCORE_FORMAT_ATTR(event14v2, event, "config:0-7,32-37"); /* PerfMonV2 DF */
DEFINE_UNCORE_FORMAT_ATTR(event8, event, "config:0-7"); /* F17h+ L3 */ DEFINE_UNCORE_FORMAT_ATTR(event8, event, "config:0-7"); /* F17h+ L3, PerfMonV2 UMC */
DEFINE_UNCORE_FORMAT_ATTR(umask8, umask, "config:8-15"); DEFINE_UNCORE_FORMAT_ATTR(umask8, umask, "config:8-15");
DEFINE_UNCORE_FORMAT_ATTR(umask12, umask, "config:8-15,24-27"); /* PerfMonV2 DF */ DEFINE_UNCORE_FORMAT_ATTR(umask12, umask, "config:8-15,24-27"); /* PerfMonV2 DF */
DEFINE_UNCORE_FORMAT_ATTR(coreid, coreid, "config:42-44"); /* F19h L3 */ DEFINE_UNCORE_FORMAT_ATTR(coreid, coreid, "config:42-44"); /* F19h L3 */
...@@ -325,6 +297,7 @@ DEFINE_UNCORE_FORMAT_ATTR(threadmask2, threadmask, "config:56-57"); /* F19h L ...@@ -325,6 +297,7 @@ DEFINE_UNCORE_FORMAT_ATTR(threadmask2, threadmask, "config:56-57"); /* F19h L
DEFINE_UNCORE_FORMAT_ATTR(enallslices, enallslices, "config:46"); /* F19h L3 */ DEFINE_UNCORE_FORMAT_ATTR(enallslices, enallslices, "config:46"); /* F19h L3 */
DEFINE_UNCORE_FORMAT_ATTR(enallcores, enallcores, "config:47"); /* F19h L3 */ DEFINE_UNCORE_FORMAT_ATTR(enallcores, enallcores, "config:47"); /* F19h L3 */
DEFINE_UNCORE_FORMAT_ATTR(sliceid, sliceid, "config:48-50"); /* F19h L3 */ DEFINE_UNCORE_FORMAT_ATTR(sliceid, sliceid, "config:48-50"); /* F19h L3 */
DEFINE_UNCORE_FORMAT_ATTR(rdwrmask, rdwrmask, "config:8-9"); /* PerfMonV2 UMC */
/* Common DF and NB attributes */ /* Common DF and NB attributes */
static struct attribute *amd_uncore_df_format_attr[] = { static struct attribute *amd_uncore_df_format_attr[] = {
...@@ -341,6 +314,13 @@ static struct attribute *amd_uncore_l3_format_attr[] = { ...@@ -341,6 +314,13 @@ static struct attribute *amd_uncore_l3_format_attr[] = {
NULL, NULL,
}; };
/* Common UMC attributes */
static struct attribute *amd_uncore_umc_format_attr[] = {
&format_attr_event8.attr, /* event */
&format_attr_rdwrmask.attr, /* rdwrmask */
NULL,
};
/* F17h unique L3 attributes */ /* F17h unique L3 attributes */
static struct attribute *amd_f17h_uncore_l3_format_attr[] = { static struct attribute *amd_f17h_uncore_l3_format_attr[] = {
&format_attr_slicemask.attr, /* slicemask */ &format_attr_slicemask.attr, /* slicemask */
...@@ -378,6 +358,11 @@ static struct attribute_group amd_f19h_uncore_l3_format_group = { ...@@ -378,6 +358,11 @@ static struct attribute_group amd_f19h_uncore_l3_format_group = {
.is_visible = amd_f19h_uncore_is_visible, .is_visible = amd_f19h_uncore_is_visible,
}; };
static struct attribute_group amd_uncore_umc_format_group = {
.name = "format",
.attrs = amd_uncore_umc_format_attr,
};
static const struct attribute_group *amd_uncore_df_attr_groups[] = { static const struct attribute_group *amd_uncore_df_attr_groups[] = {
&amd_uncore_attr_group, &amd_uncore_attr_group,
&amd_uncore_df_format_group, &amd_uncore_df_format_group,
...@@ -396,389 +381,732 @@ static const struct attribute_group *amd_uncore_l3_attr_update[] = { ...@@ -396,389 +381,732 @@ static const struct attribute_group *amd_uncore_l3_attr_update[] = {
NULL, NULL,
}; };
static struct pmu amd_nb_pmu = { static const struct attribute_group *amd_uncore_umc_attr_groups[] = {
.task_ctx_nr = perf_invalid_context, &amd_uncore_attr_group,
.attr_groups = amd_uncore_df_attr_groups, &amd_uncore_umc_format_group,
.name = "amd_nb", NULL,
.event_init = amd_uncore_event_init,
.add = amd_uncore_add,
.del = amd_uncore_del,
.start = amd_uncore_start,
.stop = amd_uncore_stop,
.read = amd_uncore_read,
.capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
.module = THIS_MODULE,
}; };
static struct pmu amd_llc_pmu = { static __always_inline
.task_ctx_nr = perf_invalid_context, int amd_uncore_ctx_cid(struct amd_uncore *uncore, unsigned int cpu)
.attr_groups = amd_uncore_l3_attr_groups, {
.attr_update = amd_uncore_l3_attr_update, union amd_uncore_info *info = per_cpu_ptr(uncore->info, cpu);
.name = "amd_l2", return info->split.cid;
.event_init = amd_uncore_event_init, }
.add = amd_uncore_add,
.del = amd_uncore_del,
.start = amd_uncore_start,
.stop = amd_uncore_stop,
.read = amd_uncore_read,
.capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
.module = THIS_MODULE,
};
static struct amd_uncore *amd_uncore_alloc(unsigned int cpu) static __always_inline
int amd_uncore_ctx_gid(struct amd_uncore *uncore, unsigned int cpu)
{ {
return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL, union amd_uncore_info *info = per_cpu_ptr(uncore->info, cpu);
cpu_to_node(cpu)); return info->split.gid;
} }
static inline struct perf_event ** static __always_inline
amd_uncore_events_alloc(unsigned int num, unsigned int cpu) int amd_uncore_ctx_num_pmcs(struct amd_uncore *uncore, unsigned int cpu)
{ {
return kzalloc_node(sizeof(struct perf_event *) * num, GFP_KERNEL, union amd_uncore_info *info = per_cpu_ptr(uncore->info, cpu);
cpu_to_node(cpu)); return info->split.num_pmcs;
} }
static int amd_uncore_cpu_up_prepare(unsigned int cpu) static void amd_uncore_ctx_free(struct amd_uncore *uncore, unsigned int cpu)
{ {
struct amd_uncore *uncore_nb = NULL, *uncore_llc = NULL; struct amd_uncore_pmu *pmu;
struct amd_uncore_ctx *ctx;
int i;
if (amd_uncore_nb) { if (!uncore->init_done)
*per_cpu_ptr(amd_uncore_nb, cpu) = NULL; return;
uncore_nb = amd_uncore_alloc(cpu);
if (!uncore_nb) for (i = 0; i < uncore->num_pmus; i++) {
goto fail; pmu = &uncore->pmus[i];
uncore_nb->cpu = cpu; ctx = *per_cpu_ptr(pmu->ctx, cpu);
uncore_nb->num_counters = num_counters_nb; if (!ctx)
uncore_nb->rdpmc_base = RDPMC_BASE_NB; continue;
uncore_nb->msr_base = MSR_F15H_NB_PERF_CTL;
uncore_nb->active_mask = &amd_nb_active_mask; if (cpu == ctx->cpu)
uncore_nb->pmu = &amd_nb_pmu; cpumask_clear_cpu(cpu, &pmu->active_mask);
uncore_nb->events = amd_uncore_events_alloc(num_counters_nb, cpu);
if (!uncore_nb->events) if (!--ctx->refcnt) {
goto fail; kfree(ctx->events);
uncore_nb->id = -1; kfree(ctx);
*per_cpu_ptr(amd_uncore_nb, cpu) = uncore_nb;
} }
if (amd_uncore_llc) { *per_cpu_ptr(pmu->ctx, cpu) = NULL;
*per_cpu_ptr(amd_uncore_llc, cpu) = NULL;
uncore_llc = amd_uncore_alloc(cpu);
if (!uncore_llc)
goto fail;
uncore_llc->cpu = cpu;
uncore_llc->num_counters = num_counters_llc;
uncore_llc->rdpmc_base = RDPMC_BASE_LLC;
uncore_llc->msr_base = MSR_F16H_L2I_PERF_CTL;
uncore_llc->active_mask = &amd_llc_active_mask;
uncore_llc->pmu = &amd_llc_pmu;
uncore_llc->events = amd_uncore_events_alloc(num_counters_llc, cpu);
if (!uncore_llc->events)
goto fail;
uncore_llc->id = -1;
*per_cpu_ptr(amd_uncore_llc, cpu) = uncore_llc;
} }
}
static int amd_uncore_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
{
struct amd_uncore_ctx *curr, *prev;
struct amd_uncore_pmu *pmu;
int node, cid, gid, i, j;
if (!uncore->init_done || !uncore->num_pmus)
return 0; return 0;
fail: cid = amd_uncore_ctx_cid(uncore, cpu);
if (uncore_nb) { gid = amd_uncore_ctx_gid(uncore, cpu);
kfree(uncore_nb->events);
kfree(uncore_nb); for (i = 0; i < uncore->num_pmus; i++) {
pmu = &uncore->pmus[i];
*per_cpu_ptr(pmu->ctx, cpu) = NULL;
curr = NULL;
/* Check for group exclusivity */
if (gid != pmu->group)
continue;
/* Find a sibling context */
for_each_online_cpu(j) {
if (cpu == j)
continue;
prev = *per_cpu_ptr(pmu->ctx, j);
if (!prev)
continue;
if (cid == amd_uncore_ctx_cid(uncore, j)) {
curr = prev;
break;
}
}
/* Allocate context if sibling does not exist */
if (!curr) {
node = cpu_to_node(cpu);
curr = kzalloc_node(sizeof(*curr), GFP_KERNEL, node);
if (!curr)
goto fail;
curr->cpu = cpu;
curr->events = kzalloc_node(sizeof(*curr->events) *
pmu->num_counters,
GFP_KERNEL, node);
if (!curr->events) {
kfree(curr);
goto fail;
} }
if (uncore_llc) { cpumask_set_cpu(cpu, &pmu->active_mask);
kfree(uncore_llc->events);
kfree(uncore_llc);
} }
curr->refcnt++;
*per_cpu_ptr(pmu->ctx, cpu) = curr;
}
return 0;
fail:
amd_uncore_ctx_free(uncore, cpu);
return -ENOMEM; return -ENOMEM;
} }
static struct amd_uncore * static void amd_uncore_ctx_move(struct amd_uncore *uncore, unsigned int cpu)
amd_uncore_find_online_sibling(struct amd_uncore *this,
struct amd_uncore * __percpu *uncores)
{ {
unsigned int cpu; struct amd_uncore_ctx *curr, *next;
struct amd_uncore *that; struct amd_uncore_pmu *pmu;
int i, j;
for_each_online_cpu(cpu) { if (!uncore->init_done)
that = *per_cpu_ptr(uncores, cpu); return;
if (!that) for (i = 0; i < uncore->num_pmus; i++) {
pmu = &uncore->pmus[i];
curr = *per_cpu_ptr(pmu->ctx, cpu);
if (!curr)
continue; continue;
if (this == that) /* Migrate to a shared sibling if possible */
for_each_online_cpu(j) {
next = *per_cpu_ptr(pmu->ctx, j);
if (!next || cpu == j)
continue; continue;
if (this->id == that->id) { if (curr == next) {
hlist_add_head(&this->node, &uncore_unused_list); perf_pmu_migrate_context(&pmu->pmu, cpu, j);
this = that; cpumask_clear_cpu(cpu, &pmu->active_mask);
cpumask_set_cpu(j, &pmu->active_mask);
next->cpu = j;
break; break;
} }
} }
}
this->refcnt++;
return this;
} }
static int amd_uncore_cpu_starting(unsigned int cpu) static int amd_uncore_cpu_starting(unsigned int cpu)
{ {
unsigned int eax, ebx, ecx, edx;
struct amd_uncore *uncore; struct amd_uncore *uncore;
int i;
if (amd_uncore_nb) { for (i = 0; i < UNCORE_TYPE_MAX; i++) {
uncore = *per_cpu_ptr(amd_uncore_nb, cpu); uncore = &uncores[i];
cpuid(0x8000001e, &eax, &ebx, &ecx, &edx); uncore->scan(uncore, cpu);
uncore->id = ecx & 0xff; }
uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_nb); return 0;
*per_cpu_ptr(amd_uncore_nb, cpu) = uncore; }
static int amd_uncore_cpu_online(unsigned int cpu)
{
struct amd_uncore *uncore;
int i;
for (i = 0; i < UNCORE_TYPE_MAX; i++) {
uncore = &uncores[i];
if (uncore->init(uncore, cpu))
break;
} }
if (amd_uncore_llc) { return 0;
uncore = *per_cpu_ptr(amd_uncore_llc, cpu); }
uncore->id = get_llc_id(cpu);
static int amd_uncore_cpu_down_prepare(unsigned int cpu)
{
struct amd_uncore *uncore;
int i;
uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_llc); for (i = 0; i < UNCORE_TYPE_MAX; i++) {
*per_cpu_ptr(amd_uncore_llc, cpu) = uncore; uncore = &uncores[i];
uncore->move(uncore, cpu);
} }
return 0; return 0;
} }
static void uncore_clean_online(void) static int amd_uncore_cpu_dead(unsigned int cpu)
{ {
struct amd_uncore *uncore; struct amd_uncore *uncore;
struct hlist_node *n; int i;
hlist_for_each_entry_safe(uncore, n, &uncore_unused_list, node) { for (i = 0; i < UNCORE_TYPE_MAX; i++) {
hlist_del(&uncore->node); uncore = &uncores[i];
kfree(uncore->events); uncore->free(uncore, cpu);
kfree(uncore);
} }
return 0;
} }
static void uncore_online(unsigned int cpu, static int amd_uncore_df_event_init(struct perf_event *event)
struct amd_uncore * __percpu *uncores)
{ {
struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu); struct hw_perf_event *hwc = &event->hw;
int ret = amd_uncore_event_init(event);
uncore_clean_online(); if (ret || pmu_version < 2)
return ret;
hwc->config = event->attr.config &
(pmu_version >= 2 ? AMD64_PERFMON_V2_RAW_EVENT_MASK_NB :
AMD64_RAW_EVENT_MASK_NB);
if (cpu == uncore->cpu) return 0;
cpumask_set_cpu(cpu, uncore->active_mask);
} }
static int amd_uncore_cpu_online(unsigned int cpu) static int amd_uncore_df_add(struct perf_event *event, int flags)
{ {
if (amd_uncore_nb) int ret = amd_uncore_add(event, flags & ~PERF_EF_START);
uncore_online(cpu, amd_uncore_nb); struct hw_perf_event *hwc = &event->hw;
if (ret)
return ret;
if (amd_uncore_llc) /*
uncore_online(cpu, amd_uncore_llc); * The first four DF counters are accessible via RDPMC index 6 to 9
* followed by the L3 counters from index 10 to 15. For processors
* with more than four DF counters, the DF RDPMC assignments become
* discontiguous as the additional counters are accessible starting
* from index 16.
*/
if (hwc->idx >= NUM_COUNTERS_NB)
hwc->event_base_rdpmc += NUM_COUNTERS_L3;
/* Delayed start after rdpmc base update */
if (flags & PERF_EF_START)
amd_uncore_start(event, PERF_EF_RELOAD);
return 0; return 0;
} }
static void uncore_down_prepare(unsigned int cpu, static
struct amd_uncore * __percpu *uncores) void amd_uncore_df_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)
{ {
unsigned int i; union cpuid_0x80000022_ebx ebx;
struct amd_uncore *this = *per_cpu_ptr(uncores, cpu); union amd_uncore_info info;
if (this->cpu != cpu) if (!boot_cpu_has(X86_FEATURE_PERFCTR_NB))
return; return;
/* this cpu is going down, migrate to a shared sibling if possible */ info.split.aux_data = 0;
for_each_online_cpu(i) { info.split.num_pmcs = NUM_COUNTERS_NB;
struct amd_uncore *that = *per_cpu_ptr(uncores, i); info.split.gid = 0;
info.split.cid = topology_die_id(cpu);
if (cpu == i) if (pmu_version >= 2) {
continue; ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES);
info.split.num_pmcs = ebx.split.num_df_pmc;
}
if (this == that) { *per_cpu_ptr(uncore->info, cpu) = info;
perf_pmu_migrate_context(this->pmu, cpu, i); }
cpumask_clear_cpu(cpu, that->active_mask);
cpumask_set_cpu(i, that->active_mask); static
that->cpu = i; int amd_uncore_df_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
break; {
struct attribute **df_attr = amd_uncore_df_format_attr;
struct amd_uncore_pmu *pmu;
/* Run just once */
if (uncore->init_done)
return amd_uncore_ctx_init(uncore, cpu);
/* No grouping, single instance for a system */
uncore->pmus = kzalloc(sizeof(*uncore->pmus), GFP_KERNEL);
if (!uncore->pmus) {
uncore->num_pmus = 0;
goto done;
}
/*
* For Family 17h and above, the Northbridge counters are repurposed
* as Data Fabric counters. The PMUs are exported based on family as
* either NB or DF.
*/
pmu = &uncore->pmus[0];
strscpy(pmu->name, boot_cpu_data.x86 >= 0x17 ? "amd_df" : "amd_nb",
sizeof(pmu->name));
pmu->num_counters = amd_uncore_ctx_num_pmcs(uncore, cpu);
pmu->msr_base = MSR_F15H_NB_PERF_CTL;
pmu->rdpmc_base = RDPMC_BASE_NB;
pmu->group = amd_uncore_ctx_gid(uncore, cpu);
if (pmu_version >= 2) {
*df_attr++ = &format_attr_event14v2.attr;
*df_attr++ = &format_attr_umask12.attr;
} else if (boot_cpu_data.x86 >= 0x17) {
*df_attr = &format_attr_event14.attr;
} }
pmu->ctx = alloc_percpu(struct amd_uncore_ctx *);
if (!pmu->ctx)
goto done;
pmu->pmu = (struct pmu) {
.task_ctx_nr = perf_invalid_context,
.attr_groups = amd_uncore_df_attr_groups,
.name = pmu->name,
.event_init = amd_uncore_df_event_init,
.add = amd_uncore_df_add,
.del = amd_uncore_del,
.start = amd_uncore_start,
.stop = amd_uncore_stop,
.read = amd_uncore_read,
.capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
.module = THIS_MODULE,
};
if (perf_pmu_register(&pmu->pmu, pmu->pmu.name, -1)) {
free_percpu(pmu->ctx);
pmu->ctx = NULL;
goto done;
} }
pr_info("%d %s%s counters detected\n", pmu->num_counters,
boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON " : "",
pmu->pmu.name);
uncore->num_pmus = 1;
done:
uncore->init_done = true;
return amd_uncore_ctx_init(uncore, cpu);
} }
static int amd_uncore_cpu_down_prepare(unsigned int cpu) static int amd_uncore_l3_event_init(struct perf_event *event)
{ {
if (amd_uncore_nb) int ret = amd_uncore_event_init(event);
uncore_down_prepare(cpu, amd_uncore_nb); struct hw_perf_event *hwc = &event->hw;
u64 config = event->attr.config;
u64 mask;
hwc->config = config & AMD64_RAW_EVENT_MASK_NB;
/*
* SliceMask and ThreadMask need to be set for certain L3 events.
* For other events, the two fields do not affect the count.
*/
if (ret || boot_cpu_data.x86 < 0x17)
return ret;
mask = config & (AMD64_L3_F19H_THREAD_MASK | AMD64_L3_SLICEID_MASK |
AMD64_L3_EN_ALL_CORES | AMD64_L3_EN_ALL_SLICES |
AMD64_L3_COREID_MASK);
if (boot_cpu_data.x86 <= 0x18)
mask = ((config & AMD64_L3_SLICE_MASK) ? : AMD64_L3_SLICE_MASK) |
((config & AMD64_L3_THREAD_MASK) ? : AMD64_L3_THREAD_MASK);
/*
* If the user doesn't specify a ThreadMask, they're not trying to
* count core 0, so we enable all cores & threads.
* We'll also assume that they want to count slice 0 if they specify
* a ThreadMask and leave SliceId and EnAllSlices unpopulated.
*/
else if (!(config & AMD64_L3_F19H_THREAD_MASK))
mask = AMD64_L3_F19H_THREAD_MASK | AMD64_L3_EN_ALL_SLICES |
AMD64_L3_EN_ALL_CORES;
if (amd_uncore_llc) hwc->config |= mask;
uncore_down_prepare(cpu, amd_uncore_llc);
return 0; return 0;
} }
static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores) static
void amd_uncore_l3_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)
{ {
struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu); union amd_uncore_info info;
if (cpu == uncore->cpu) if (!boot_cpu_has(X86_FEATURE_PERFCTR_LLC))
cpumask_clear_cpu(cpu, uncore->active_mask); return;
info.split.aux_data = 0;
info.split.num_pmcs = NUM_COUNTERS_L2;
info.split.gid = 0;
info.split.cid = get_llc_id(cpu);
if (boot_cpu_data.x86 >= 0x17)
info.split.num_pmcs = NUM_COUNTERS_L3;
*per_cpu_ptr(uncore->info, cpu) = info;
}
static
int amd_uncore_l3_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
{
struct attribute **l3_attr = amd_uncore_l3_format_attr;
struct amd_uncore_pmu *pmu;
/* Run just once */
if (uncore->init_done)
return amd_uncore_ctx_init(uncore, cpu);
/* No grouping, single instance for a system */
uncore->pmus = kzalloc(sizeof(*uncore->pmus), GFP_KERNEL);
if (!uncore->pmus) {
uncore->num_pmus = 0;
goto done;
}
/*
* For Family 17h and above, L3 cache counters are available instead
* of L2 cache counters. The PMUs are exported based on family as
* either L2 or L3.
*/
pmu = &uncore->pmus[0];
strscpy(pmu->name, boot_cpu_data.x86 >= 0x17 ? "amd_l3" : "amd_l2",
sizeof(pmu->name));
pmu->num_counters = amd_uncore_ctx_num_pmcs(uncore, cpu);
pmu->msr_base = MSR_F16H_L2I_PERF_CTL;
pmu->rdpmc_base = RDPMC_BASE_LLC;
pmu->group = amd_uncore_ctx_gid(uncore, cpu);
if (boot_cpu_data.x86 >= 0x17) {
*l3_attr++ = &format_attr_event8.attr;
*l3_attr++ = &format_attr_umask8.attr;
*l3_attr++ = boot_cpu_data.x86 >= 0x19 ?
&format_attr_threadmask2.attr :
&format_attr_threadmask8.attr;
}
pmu->ctx = alloc_percpu(struct amd_uncore_ctx *);
if (!pmu->ctx)
goto done;
pmu->pmu = (struct pmu) {
.task_ctx_nr = perf_invalid_context,
.attr_groups = amd_uncore_l3_attr_groups,
.attr_update = amd_uncore_l3_attr_update,
.name = pmu->name,
.event_init = amd_uncore_l3_event_init,
.add = amd_uncore_add,
.del = amd_uncore_del,
.start = amd_uncore_start,
.stop = amd_uncore_stop,
.read = amd_uncore_read,
.capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
.module = THIS_MODULE,
};
if (!--uncore->refcnt) { if (perf_pmu_register(&pmu->pmu, pmu->pmu.name, -1)) {
kfree(uncore->events); free_percpu(pmu->ctx);
kfree(uncore); pmu->ctx = NULL;
goto done;
} }
*per_cpu_ptr(uncores, cpu) = NULL; pr_info("%d %s%s counters detected\n", pmu->num_counters,
boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON " : "",
pmu->pmu.name);
uncore->num_pmus = 1;
done:
uncore->init_done = true;
return amd_uncore_ctx_init(uncore, cpu);
} }
static int amd_uncore_cpu_dead(unsigned int cpu) static int amd_uncore_umc_event_init(struct perf_event *event)
{ {
if (amd_uncore_nb) struct hw_perf_event *hwc = &event->hw;
uncore_dead(cpu, amd_uncore_nb); int ret = amd_uncore_event_init(event);
if (ret)
return ret;
if (amd_uncore_llc) hwc->config = event->attr.config & AMD64_PERFMON_V2_RAW_EVENT_MASK_UMC;
uncore_dead(cpu, amd_uncore_llc);
return 0; return 0;
} }
static int __init amd_uncore_init(void) static void amd_uncore_umc_start(struct perf_event *event, int flags)
{
struct hw_perf_event *hwc = &event->hw;
if (flags & PERF_EF_RELOAD)
wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
hwc->state = 0;
wrmsrl(hwc->config_base, (hwc->config | AMD64_PERFMON_V2_ENABLE_UMC));
perf_event_update_userpage(event);
}
static
void amd_uncore_umc_ctx_scan(struct amd_uncore *uncore, unsigned int cpu)
{ {
struct attribute **df_attr = amd_uncore_df_format_attr;
struct attribute **l3_attr = amd_uncore_l3_format_attr;
union cpuid_0x80000022_ebx ebx; union cpuid_0x80000022_ebx ebx;
int ret = -ENODEV; union amd_uncore_info info;
unsigned int eax, ecx, edx;
if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && if (pmu_version < 2)
boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) return;
return -ENODEV;
if (!boot_cpu_has(X86_FEATURE_TOPOEXT)) cpuid(EXT_PERFMON_DEBUG_FEATURES, &eax, &ebx.full, &ecx, &edx);
return -ENODEV; info.split.aux_data = ecx; /* stash active mask */
info.split.num_pmcs = ebx.split.num_umc_pmc;
info.split.gid = topology_die_id(cpu);
info.split.cid = topology_die_id(cpu);
*per_cpu_ptr(uncore->info, cpu) = info;
}
if (boot_cpu_has(X86_FEATURE_PERFMON_V2)) static
pmu_version = 2; int amd_uncore_umc_ctx_init(struct amd_uncore *uncore, unsigned int cpu)
{
DECLARE_BITMAP(gmask, UNCORE_GROUP_MAX) = { 0 };
u8 group_num_pmus[UNCORE_GROUP_MAX] = { 0 };
u8 group_num_pmcs[UNCORE_GROUP_MAX] = { 0 };
union amd_uncore_info info;
struct amd_uncore_pmu *pmu;
int index = 0, gid, i;
if (pmu_version < 2)
return 0;
num_counters_nb = NUM_COUNTERS_NB; /* Run just once */
num_counters_llc = NUM_COUNTERS_L2; if (uncore->init_done)
if (boot_cpu_data.x86 >= 0x17) { return amd_uncore_ctx_init(uncore, cpu);
/*
* For F17h and above, the Northbridge counters are
* repurposed as Data Fabric counters. Also, L3
* counters are supported too. The PMUs are exported
* based on family as either L2 or L3 and NB or DF.
*/
num_counters_llc = NUM_COUNTERS_L3;
amd_nb_pmu.name = "amd_df";
amd_llc_pmu.name = "amd_l3";
l3_mask = true;
}
if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) { /* Find unique groups */
if (pmu_version >= 2) { for_each_online_cpu(i) {
*df_attr++ = &format_attr_event14v2.attr; info = *per_cpu_ptr(uncore->info, i);
*df_attr++ = &format_attr_umask12.attr; gid = info.split.gid;
} else if (boot_cpu_data.x86 >= 0x17) { if (test_bit(gid, gmask))
*df_attr = &format_attr_event14.attr; continue;
}
amd_uncore_nb = alloc_percpu(struct amd_uncore *); __set_bit(gid, gmask);
if (!amd_uncore_nb) { group_num_pmus[gid] = hweight32(info.split.aux_data);
ret = -ENOMEM; group_num_pmcs[gid] = info.split.num_pmcs;
goto fail_nb; uncore->num_pmus += group_num_pmus[gid];
} }
ret = perf_pmu_register(&amd_nb_pmu, amd_nb_pmu.name, -1);
if (ret)
goto fail_nb;
if (pmu_version >= 2) { uncore->pmus = kzalloc(sizeof(*uncore->pmus) * uncore->num_pmus,
ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES); GFP_KERNEL);
num_counters_nb = ebx.split.num_df_pmc; if (!uncore->pmus) {
uncore->num_pmus = 0;
goto done;
} }
pr_info("%d %s %s counters detected\n", num_counters_nb, for_each_set_bit(gid, gmask, UNCORE_GROUP_MAX) {
boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON" : "", for (i = 0; i < group_num_pmus[gid]; i++) {
amd_nb_pmu.name); pmu = &uncore->pmus[index];
snprintf(pmu->name, sizeof(pmu->name), "amd_umc_%d", index);
pmu->num_counters = group_num_pmcs[gid] / group_num_pmus[gid];
pmu->msr_base = MSR_F19H_UMC_PERF_CTL + i * pmu->num_counters * 2;
pmu->rdpmc_base = -1;
pmu->group = gid;
ret = 0; pmu->ctx = alloc_percpu(struct amd_uncore_ctx *);
} if (!pmu->ctx)
goto done;
if (boot_cpu_has(X86_FEATURE_PERFCTR_LLC)) { pmu->pmu = (struct pmu) {
if (boot_cpu_data.x86 >= 0x19) { .task_ctx_nr = perf_invalid_context,
*l3_attr++ = &format_attr_event8.attr; .attr_groups = amd_uncore_umc_attr_groups,
*l3_attr++ = &format_attr_umask8.attr; .name = pmu->name,
*l3_attr++ = &format_attr_threadmask2.attr; .event_init = amd_uncore_umc_event_init,
} else if (boot_cpu_data.x86 >= 0x17) { .add = amd_uncore_add,
*l3_attr++ = &format_attr_event8.attr; .del = amd_uncore_del,
*l3_attr++ = &format_attr_umask8.attr; .start = amd_uncore_umc_start,
*l3_attr++ = &format_attr_threadmask8.attr; .stop = amd_uncore_stop,
.read = amd_uncore_read,
.capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT,
.module = THIS_MODULE,
};
if (perf_pmu_register(&pmu->pmu, pmu->pmu.name, -1)) {
free_percpu(pmu->ctx);
pmu->ctx = NULL;
goto done;
} }
amd_uncore_llc = alloc_percpu(struct amd_uncore *); pr_info("%d %s counters detected\n", pmu->num_counters,
if (!amd_uncore_llc) { pmu->pmu.name);
ret = -ENOMEM;
goto fail_llc; index++;
} }
ret = perf_pmu_register(&amd_llc_pmu, amd_llc_pmu.name, -1); }
if (ret)
goto fail_llc; done:
uncore->num_pmus = index;
uncore->init_done = true;
return amd_uncore_ctx_init(uncore, cpu);
}
static struct amd_uncore uncores[UNCORE_TYPE_MAX] = {
/* UNCORE_TYPE_DF */
{
.scan = amd_uncore_df_ctx_scan,
.init = amd_uncore_df_ctx_init,
.move = amd_uncore_ctx_move,
.free = amd_uncore_ctx_free,
},
/* UNCORE_TYPE_L3 */
{
.scan = amd_uncore_l3_ctx_scan,
.init = amd_uncore_l3_ctx_init,
.move = amd_uncore_ctx_move,
.free = amd_uncore_ctx_free,
},
/* UNCORE_TYPE_UMC */
{
.scan = amd_uncore_umc_ctx_scan,
.init = amd_uncore_umc_ctx_init,
.move = amd_uncore_ctx_move,
.free = amd_uncore_ctx_free,
},
};
static int __init amd_uncore_init(void)
{
struct amd_uncore *uncore;
int ret = -ENODEV;
int i;
if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
return -ENODEV;
if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
return -ENODEV;
pr_info("%d %s %s counters detected\n", num_counters_llc, if (boot_cpu_has(X86_FEATURE_PERFMON_V2))
boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON" : "", pmu_version = 2;
amd_llc_pmu.name);
ret = 0; for (i = 0; i < UNCORE_TYPE_MAX; i++) {
uncore = &uncores[i];
BUG_ON(!uncore->scan);
BUG_ON(!uncore->init);
BUG_ON(!uncore->move);
BUG_ON(!uncore->free);
uncore->info = alloc_percpu(union amd_uncore_info);
if (!uncore->info) {
ret = -ENOMEM;
goto fail;
} }
};
/* /*
* Install callbacks. Core will call them for each online cpu. * Install callbacks. Core will call them for each online cpu.
*/ */
if (cpuhp_setup_state(CPUHP_PERF_X86_AMD_UNCORE_PREP, ret = cpuhp_setup_state(CPUHP_PERF_X86_AMD_UNCORE_PREP,
"perf/x86/amd/uncore:prepare", "perf/x86/amd/uncore:prepare",
amd_uncore_cpu_up_prepare, amd_uncore_cpu_dead)) NULL, amd_uncore_cpu_dead);
goto fail_llc; if (ret)
goto fail;
if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING, ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING,
"perf/x86/amd/uncore:starting", "perf/x86/amd/uncore:starting",
amd_uncore_cpu_starting, NULL)) amd_uncore_cpu_starting, NULL);
if (ret)
goto fail_prep; goto fail_prep;
if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE,
ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE,
"perf/x86/amd/uncore:online", "perf/x86/amd/uncore:online",
amd_uncore_cpu_online, amd_uncore_cpu_online,
amd_uncore_cpu_down_prepare)) amd_uncore_cpu_down_prepare);
if (ret)
goto fail_start; goto fail_start;
return 0; return 0;
fail_start: fail_start:
cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING); cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
fail_prep: fail_prep:
cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP); cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP);
fail_llc: fail:
if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) for (i = 0; i < UNCORE_TYPE_MAX; i++) {
perf_pmu_unregister(&amd_nb_pmu); uncore = &uncores[i];
free_percpu(amd_uncore_llc); if (uncore->info) {
fail_nb: free_percpu(uncore->info);
free_percpu(amd_uncore_nb); uncore->info = NULL;
}
}
return ret; return ret;
} }
static void __exit amd_uncore_exit(void) static void __exit amd_uncore_exit(void)
{ {
struct amd_uncore *uncore;
struct amd_uncore_pmu *pmu;
int i, j;
cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE); cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE);
cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING); cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING);
cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP); cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP);
if (boot_cpu_has(X86_FEATURE_PERFCTR_LLC)) { for (i = 0; i < UNCORE_TYPE_MAX; i++) {
perf_pmu_unregister(&amd_llc_pmu); uncore = &uncores[i];
free_percpu(amd_uncore_llc); if (!uncore->info)
amd_uncore_llc = NULL; continue;
free_percpu(uncore->info);
uncore->info = NULL;
for (j = 0; j < uncore->num_pmus; j++) {
pmu = &uncore->pmus[j];
if (!pmu->ctx)
continue;
perf_pmu_unregister(&pmu->pmu);
free_percpu(pmu->ctx);
pmu->ctx = NULL;
} }
if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) { kfree(uncore->pmus);
perf_pmu_unregister(&amd_nb_pmu); uncore->pmus = NULL;
free_percpu(amd_uncore_nb);
amd_uncore_nb = NULL;
} }
} }
......
arch/x86/events/core.c
View file @ bceb7acc
...@@ -1887,9 +1887,9 @@ ssize_t events_hybrid_sysfs_show(struct device *dev, ...@@ -1887,9 +1887,9 @@ ssize_t events_hybrid_sysfs_show(struct device *dev,
str = pmu_attr->event_str; str = pmu_attr->event_str;
for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) { for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) {
if (!(x86_pmu.hybrid_pmu[i].cpu_type & pmu_attr->pmu_type)) if (!(x86_pmu.hybrid_pmu[i].pmu_type & pmu_attr->pmu_type))
continue; continue;
if (x86_pmu.hybrid_pmu[i].cpu_type & pmu->cpu_type) { if (x86_pmu.hybrid_pmu[i].pmu_type & pmu->pmu_type) {
next_str = strchr(str, ';'); next_str = strchr(str, ';');
if (next_str) if (next_str)
return snprintf(page, next_str - str + 1, "%s", str); return snprintf(page, next_str - str + 1, "%s", str);
...@@ -2169,7 +2169,7 @@ static int __init init_hw_perf_events(void) ...@@ -2169,7 +2169,7 @@ static int __init init_hw_perf_events(void)
hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_EXTENDED_HW_TYPE; hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_EXTENDED_HW_TYPE;
err = perf_pmu_register(&hybrid_pmu->pmu, hybrid_pmu->name, err = perf_pmu_register(&hybrid_pmu->pmu, hybrid_pmu->name,
(hybrid_pmu->cpu_type == hybrid_big) ? PERF_TYPE_RAW : -1); (hybrid_pmu->pmu_type == hybrid_big) ? PERF_TYPE_RAW : -1);
if (err) if (err)
break; break;
} }
......
arch/x86/events/intel/core.c
View file @ bceb7acc
...@@ -211,6 +211,14 @@ static struct event_constraint intel_slm_event_constraints[] __read_mostly = ...@@ -211,6 +211,14 @@ static struct event_constraint intel_slm_event_constraints[] __read_mostly =
EVENT_CONSTRAINT_END EVENT_CONSTRAINT_END
}; };
static struct event_constraint intel_grt_event_constraints[] __read_mostly = {
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF */
FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */
EVENT_CONSTRAINT_END
};
static struct event_constraint intel_skl_event_constraints[] = { static struct event_constraint intel_skl_event_constraints[] = {
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
...@@ -299,7 +307,7 @@ static struct extra_reg intel_icl_extra_regs[] __read_mostly = { ...@@ -299,7 +307,7 @@ static struct extra_reg intel_icl_extra_regs[] __read_mostly = {
EVENT_EXTRA_END EVENT_EXTRA_END
}; };
static struct extra_reg intel_spr_extra_regs[] __read_mostly = { static struct extra_reg intel_glc_extra_regs[] __read_mostly = {
INTEL_UEVENT_EXTRA_REG(0x012a, MSR_OFFCORE_RSP_0, 0x3fffffffffull, RSP_0), INTEL_UEVENT_EXTRA_REG(0x012a, MSR_OFFCORE_RSP_0, 0x3fffffffffull, RSP_0),
INTEL_UEVENT_EXTRA_REG(0x012b, MSR_OFFCORE_RSP_1, 0x3fffffffffull, RSP_1), INTEL_UEVENT_EXTRA_REG(0x012b, MSR_OFFCORE_RSP_1, 0x3fffffffffull, RSP_1),
INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd), INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
...@@ -309,11 +317,12 @@ static struct extra_reg intel_spr_extra_regs[] __read_mostly = { ...@@ -309,11 +317,12 @@ static struct extra_reg intel_spr_extra_regs[] __read_mostly = {
EVENT_EXTRA_END EVENT_EXTRA_END
}; };
static struct event_constraint intel_spr_event_constraints[] = { static struct event_constraint intel_glc_event_constraints[] = {
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */ FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x0100, 0), /* INST_RETIRED.PREC_DIST */ FIXED_EVENT_CONSTRAINT(0x0100, 0), /* INST_RETIRED.PREC_DIST */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */ FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */ FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */
FIXED_EVENT_CONSTRAINT(0x0400, 3), /* SLOTS */ FIXED_EVENT_CONSTRAINT(0x0400, 3), /* SLOTS */
METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0), METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0),
METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1), METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1),
...@@ -349,7 +358,7 @@ static struct event_constraint intel_spr_event_constraints[] = { ...@@ -349,7 +358,7 @@ static struct event_constraint intel_spr_event_constraints[] = {
EVENT_CONSTRAINT_END EVENT_CONSTRAINT_END
}; };
static struct extra_reg intel_gnr_extra_regs[] __read_mostly = { static struct extra_reg intel_rwc_extra_regs[] __read_mostly = {
INTEL_UEVENT_EXTRA_REG(0x012a, MSR_OFFCORE_RSP_0, 0x3fffffffffull, RSP_0), INTEL_UEVENT_EXTRA_REG(0x012a, MSR_OFFCORE_RSP_0, 0x3fffffffffull, RSP_0),
INTEL_UEVENT_EXTRA_REG(0x012b, MSR_OFFCORE_RSP_1, 0x3fffffffffull, RSP_1), INTEL_UEVENT_EXTRA_REG(0x012b, MSR_OFFCORE_RSP_1, 0x3fffffffffull, RSP_1),
INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd), INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
...@@ -473,7 +482,7 @@ static u64 intel_pmu_event_map(int hw_event) ...@@ -473,7 +482,7 @@ static u64 intel_pmu_event_map(int hw_event)
return intel_perfmon_event_map[hw_event]; return intel_perfmon_event_map[hw_event];
} }
static __initconst const u64 spr_hw_cache_event_ids static __initconst const u64 glc_hw_cache_event_ids
[PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_MAX]
[PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_OP_MAX]
[PERF_COUNT_HW_CACHE_RESULT_MAX] = [PERF_COUNT_HW_CACHE_RESULT_MAX] =
...@@ -552,7 +561,7 @@ static __initconst const u64 spr_hw_cache_event_ids ...@@ -552,7 +561,7 @@ static __initconst const u64 spr_hw_cache_event_ids
}, },
}; };
static __initconst const u64 spr_hw_cache_extra_regs static __initconst const u64 glc_hw_cache_extra_regs
[PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_MAX]
[PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_OP_MAX]
[PERF_COUNT_HW_CACHE_RESULT_MAX] = [PERF_COUNT_HW_CACHE_RESULT_MAX] =
...@@ -2556,16 +2565,6 @@ static int icl_set_topdown_event_period(struct perf_event *event) ...@@ -2556,16 +2565,6 @@ static int icl_set_topdown_event_period(struct perf_event *event)
return 0; return 0;
} }
static int adl_set_topdown_event_period(struct perf_event *event)
{
struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);
if (pmu->cpu_type != hybrid_big)
return 0;
return icl_set_topdown_event_period(event);
}
DEFINE_STATIC_CALL(intel_pmu_set_topdown_event_period, x86_perf_event_set_period); DEFINE_STATIC_CALL(intel_pmu_set_topdown_event_period, x86_perf_event_set_period);
static inline u64 icl_get_metrics_event_value(u64 metric, u64 slots, int idx) static inline u64 icl_get_metrics_event_value(u64 metric, u64 slots, int idx)
...@@ -2708,16 +2707,6 @@ static u64 icl_update_topdown_event(struct perf_event *event) ...@@ -2708,16 +2707,6 @@ static u64 icl_update_topdown_event(struct perf_event *event)
x86_pmu.num_topdown_events - 1); x86_pmu.num_topdown_events - 1);
} }
static u64 adl_update_topdown_event(struct perf_event *event)
{
struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);
if (pmu->cpu_type != hybrid_big)
return 0;
return icl_update_topdown_event(event);
}
DEFINE_STATIC_CALL(intel_pmu_update_topdown_event, x86_perf_event_update); DEFINE_STATIC_CALL(intel_pmu_update_topdown_event, x86_perf_event_update);
static void intel_pmu_read_topdown_event(struct perf_event *event) static void intel_pmu_read_topdown_event(struct perf_event *event)
...@@ -3869,7 +3858,7 @@ static inline bool require_mem_loads_aux_event(struct perf_event *event) ...@@ -3869,7 +3858,7 @@ static inline bool require_mem_loads_aux_event(struct perf_event *event)
return false; return false;
if (is_hybrid()) if (is_hybrid())
return hybrid_pmu(event->pmu)->cpu_type == hybrid_big; return hybrid_pmu(event->pmu)->pmu_type == hybrid_big;
return true; return true;
} }
...@@ -4273,7 +4262,7 @@ icl_get_event_constraints(struct cpu_hw_events *cpuc, int idx, ...@@ -4273,7 +4262,7 @@ icl_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
} }
static struct event_constraint * static struct event_constraint *
spr_get_event_constraints(struct cpu_hw_events *cpuc, int idx, glc_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
struct perf_event *event) struct perf_event *event)
{ {
struct event_constraint *c; struct event_constraint *c;
...@@ -4361,9 +4350,9 @@ adl_get_event_constraints(struct cpu_hw_events *cpuc, int idx, ...@@ -4361,9 +4350,9 @@ adl_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
{ {
struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu); struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);
if (pmu->cpu_type == hybrid_big) if (pmu->pmu_type == hybrid_big)
return spr_get_event_constraints(cpuc, idx, event); return glc_get_event_constraints(cpuc, idx, event);
else if (pmu->cpu_type == hybrid_small) else if (pmu->pmu_type == hybrid_small)
return tnt_get_event_constraints(cpuc, idx, event); return tnt_get_event_constraints(cpuc, idx, event);
WARN_ON(1); WARN_ON(1);
...@@ -4409,7 +4398,7 @@ rwc_get_event_constraints(struct cpu_hw_events *cpuc, int idx, ...@@ -4409,7 +4398,7 @@ rwc_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
{ {
struct event_constraint *c; struct event_constraint *c;
c = spr_get_event_constraints(cpuc, idx, event); c = glc_get_event_constraints(cpuc, idx, event);
/* The Retire Latency is not supported by the fixed counter 0. */ /* The Retire Latency is not supported by the fixed counter 0. */
if (event->attr.precise_ip && if (event->attr.precise_ip &&
...@@ -4433,9 +4422,9 @@ mtl_get_event_constraints(struct cpu_hw_events *cpuc, int idx, ...@@ -4433,9 +4422,9 @@ mtl_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
{ {
struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu); struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);
if (pmu->cpu_type == hybrid_big) if (pmu->pmu_type == hybrid_big)
return rwc_get_event_constraints(cpuc, idx, event); return rwc_get_event_constraints(cpuc, idx, event);
if (pmu->cpu_type == hybrid_small) if (pmu->pmu_type == hybrid_small)
return cmt_get_event_constraints(cpuc, idx, event); return cmt_get_event_constraints(cpuc, idx, event);
WARN_ON(1); WARN_ON(1);
...@@ -4446,18 +4435,18 @@ static int adl_hw_config(struct perf_event *event) ...@@ -4446,18 +4435,18 @@ static int adl_hw_config(struct perf_event *event)
{ {
struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu); struct x86_hybrid_pmu *pmu = hybrid_pmu(event->pmu);
if (pmu->cpu_type == hybrid_big) if (pmu->pmu_type == hybrid_big)
return hsw_hw_config(event); return hsw_hw_config(event);
else if (pmu->cpu_type == hybrid_small) else if (pmu->pmu_type == hybrid_small)
return intel_pmu_hw_config(event); return intel_pmu_hw_config(event);
WARN_ON(1); WARN_ON(1);
return -EOPNOTSUPP; return -EOPNOTSUPP;
} }
static u8 adl_get_hybrid_cpu_type(void) static enum hybrid_cpu_type adl_get_hybrid_cpu_type(void)
{ {
return hybrid_big; return HYBRID_INTEL_CORE;
} }
/* /*
...@@ -4490,7 +4479,7 @@ static void nhm_limit_period(struct perf_event *event, s64 *left) ...@@ -4490,7 +4479,7 @@ static void nhm_limit_period(struct perf_event *event, s64 *left)
*left = max(*left, 32LL); *left = max(*left, 32LL);
} }
static void spr_limit_period(struct perf_event *event, s64 *left) static void glc_limit_period(struct perf_event *event, s64 *left)
{ {
if (event->attr.precise_ip == 3) if (event->attr.precise_ip == 3)
*left = max(*left, 128LL); *left = max(*left, 128LL);
...@@ -4618,6 +4607,23 @@ static void intel_pmu_check_num_counters(int *num_counters, ...@@ -4618,6 +4607,23 @@ static void intel_pmu_check_num_counters(int *num_counters,
int *num_counters_fixed, int *num_counters_fixed,
u64 *intel_ctrl, u64 fixed_mask); u64 *intel_ctrl, u64 fixed_mask);
static void intel_pmu_check_event_constraints(struct event_constraint *event_constraints,
int num_counters,
int num_counters_fixed,
u64 intel_ctrl);
static void intel_pmu_check_extra_regs(struct extra_reg *extra_regs);
static inline bool intel_pmu_broken_perf_cap(void)
{
/* The Perf Metric (Bit 15) is always cleared */
if ((boot_cpu_data.x86_model == INTEL_FAM6_METEORLAKE) ||
(boot_cpu_data.x86_model == INTEL_FAM6_METEORLAKE_L))
return true;
return false;
}
static void update_pmu_cap(struct x86_hybrid_pmu *pmu) static void update_pmu_cap(struct x86_hybrid_pmu *pmu)
{ {
unsigned int sub_bitmaps = cpuid_eax(ARCH_PERFMON_EXT_LEAF); unsigned int sub_bitmaps = cpuid_eax(ARCH_PERFMON_EXT_LEAF);
...@@ -4628,27 +4634,83 @@ static void update_pmu_cap(struct x86_hybrid_pmu *pmu) ...@@ -4628,27 +4634,83 @@ static void update_pmu_cap(struct x86_hybrid_pmu *pmu)
&eax, &ebx, &ecx, &edx); &eax, &ebx, &ecx, &edx);
pmu->num_counters = fls(eax); pmu->num_counters = fls(eax);
pmu->num_counters_fixed = fls(ebx); pmu->num_counters_fixed = fls(ebx);
intel_pmu_check_num_counters(&pmu->num_counters, &pmu->num_counters_fixed, }
&pmu->intel_ctrl, ebx);
if (!intel_pmu_broken_perf_cap()) {
/* Perf Metric (Bit 15) and PEBS via PT (Bit 16) are hybrid enumeration */
rdmsrl(MSR_IA32_PERF_CAPABILITIES, pmu->intel_cap.capabilities);
} }
} }
static bool init_hybrid_pmu(int cpu) static void intel_pmu_check_hybrid_pmus(struct x86_hybrid_pmu *pmu)
{
intel_pmu_check_num_counters(&pmu->num_counters, &pmu->num_counters_fixed,
&pmu->intel_ctrl, (1ULL << pmu->num_counters_fixed) - 1);
pmu->max_pebs_events = min_t(unsigned, MAX_PEBS_EVENTS, pmu->num_counters);
pmu->unconstrained = (struct event_constraint)
__EVENT_CONSTRAINT(0, (1ULL << pmu->num_counters) - 1,
0, pmu->num_counters, 0, 0);
if (pmu->intel_cap.perf_metrics)
pmu->intel_ctrl |= 1ULL << GLOBAL_CTRL_EN_PERF_METRICS;
else
pmu->intel_ctrl &= ~(1ULL << GLOBAL_CTRL_EN_PERF_METRICS);
if (pmu->intel_cap.pebs_output_pt_available)
pmu->pmu.capabilities |= PERF_PMU_CAP_AUX_OUTPUT;
else
pmu->pmu.capabilities |= ~PERF_PMU_CAP_AUX_OUTPUT;
intel_pmu_check_event_constraints(pmu->event_constraints,
pmu->num_counters,
pmu->num_counters_fixed,
pmu->intel_ctrl);
intel_pmu_check_extra_regs(pmu->extra_regs);
}
static struct x86_hybrid_pmu *find_hybrid_pmu_for_cpu(void)
{ {
struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
u8 cpu_type = get_this_hybrid_cpu_type(); u8 cpu_type = get_this_hybrid_cpu_type();
struct x86_hybrid_pmu *pmu = NULL;
int i; int i;
if (!cpu_type && x86_pmu.get_hybrid_cpu_type) /*
* This is running on a CPU model that is known to have hybrid
* configurations. But the CPU told us it is not hybrid, shame
* on it. There should be a fixup function provided for these
* troublesome CPUs (->get_hybrid_cpu_type).
*/
if (cpu_type == HYBRID_INTEL_NONE) {
if (x86_pmu.get_hybrid_cpu_type)
cpu_type = x86_pmu.get_hybrid_cpu_type(); cpu_type = x86_pmu.get_hybrid_cpu_type();
else
return NULL;
}
/*
* This essentially just maps between the 'hybrid_cpu_type'
* and 'hybrid_pmu_type' enums:
*/
for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) { for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) {
if (x86_pmu.hybrid_pmu[i].cpu_type == cpu_type) { enum hybrid_pmu_type pmu_type = x86_pmu.hybrid_pmu[i].pmu_type;
pmu = &x86_pmu.hybrid_pmu[i];
break; if (cpu_type == HYBRID_INTEL_CORE &&
} pmu_type == hybrid_big)
return &x86_pmu.hybrid_pmu[i];
if (cpu_type == HYBRID_INTEL_ATOM &&
pmu_type == hybrid_small)
return &x86_pmu.hybrid_pmu[i];
} }
return NULL;
}
static bool init_hybrid_pmu(int cpu)
{
struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
struct x86_hybrid_pmu *pmu = find_hybrid_pmu_for_cpu();
if (WARN_ON_ONCE(!pmu || (pmu->pmu.type == -1))) { if (WARN_ON_ONCE(!pmu || (pmu->pmu.type == -1))) {
cpuc->pmu = NULL; cpuc->pmu = NULL;
return false; return false;
...@@ -4661,6 +4723,8 @@ static bool init_hybrid_pmu(int cpu) ...@@ -4661,6 +4723,8 @@ static bool init_hybrid_pmu(int cpu)
if (this_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT)) if (this_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT))
update_pmu_cap(pmu); update_pmu_cap(pmu);
intel_pmu_check_hybrid_pmus(pmu);
if (!check_hw_exists(&pmu->pmu, pmu->num_counters, pmu->num_counters_fixed)) if (!check_hw_exists(&pmu->pmu, pmu->num_counters, pmu->num_counters_fixed))
return false; return false;
...@@ -5337,14 +5401,14 @@ static struct attribute *icl_tsx_events_attrs[] = { ...@@ -5337,14 +5401,14 @@ static struct attribute *icl_tsx_events_attrs[] = {
EVENT_ATTR_STR(mem-stores, mem_st_spr, "event=0xcd,umask=0x2"); EVENT_ATTR_STR(mem-stores, mem_st_spr, "event=0xcd,umask=0x2");
EVENT_ATTR_STR(mem-loads-aux, mem_ld_aux, "event=0x03,umask=0x82"); EVENT_ATTR_STR(mem-loads-aux, mem_ld_aux, "event=0x03,umask=0x82");
static struct attribute *spr_events_attrs[] = { static struct attribute *glc_events_attrs[] = {
EVENT_PTR(mem_ld_hsw), EVENT_PTR(mem_ld_hsw),
EVENT_PTR(mem_st_spr), EVENT_PTR(mem_st_spr),
EVENT_PTR(mem_ld_aux), EVENT_PTR(mem_ld_aux),
NULL, NULL,
}; };
static struct attribute *spr_td_events_attrs[] = { static struct attribute *glc_td_events_attrs[] = {
EVENT_PTR(slots), EVENT_PTR(slots),
EVENT_PTR(td_retiring), EVENT_PTR(td_retiring),
EVENT_PTR(td_bad_spec), EVENT_PTR(td_bad_spec),
...@@ -5357,7 +5421,7 @@ static struct attribute *spr_td_events_attrs[] = { ...@@ -5357,7 +5421,7 @@ static struct attribute *spr_td_events_attrs[] = {
NULL, NULL,
}; };
static struct attribute *spr_tsx_events_attrs[] = { static struct attribute *glc_tsx_events_attrs[] = {
EVENT_PTR(tx_start), EVENT_PTR(tx_start),
EVENT_PTR(tx_abort), EVENT_PTR(tx_abort),
EVENT_PTR(tx_commit), EVENT_PTR(tx_commit),
...@@ -5699,7 +5763,7 @@ static bool is_attr_for_this_pmu(struct kobject *kobj, struct attribute *attr) ...@@ -5699,7 +5763,7 @@ static bool is_attr_for_this_pmu(struct kobject *kobj, struct attribute *attr)
struct perf_pmu_events_hybrid_attr *pmu_attr = struct perf_pmu_events_hybrid_attr *pmu_attr =
container_of(attr, struct perf_pmu_events_hybrid_attr, attr.attr); container_of(attr, struct perf_pmu_events_hybrid_attr, attr.attr);
return pmu->cpu_type & pmu_attr->pmu_type; return pmu->pmu_type & pmu_attr->pmu_type;
} }
static umode_t hybrid_events_is_visible(struct kobject *kobj, static umode_t hybrid_events_is_visible(struct kobject *kobj,
...@@ -5736,7 +5800,7 @@ static umode_t hybrid_format_is_visible(struct kobject *kobj, ...@@ -5736,7 +5800,7 @@ static umode_t hybrid_format_is_visible(struct kobject *kobj,
container_of(attr, struct perf_pmu_format_hybrid_attr, attr.attr); container_of(attr, struct perf_pmu_format_hybrid_attr, attr.attr);
int cpu = hybrid_find_supported_cpu(pmu); int cpu = hybrid_find_supported_cpu(pmu);
return (cpu >= 0) && (pmu->cpu_type & pmu_attr->pmu_type) ? attr->mode : 0; return (cpu >= 0) && (pmu->pmu_type & pmu_attr->pmu_type) ? attr->mode : 0;
} }
static struct attribute_group hybrid_group_events_td = { static struct attribute_group hybrid_group_events_td = {
...@@ -5880,40 +5944,105 @@ static void intel_pmu_check_extra_regs(struct extra_reg *extra_regs) ...@@ -5880,40 +5944,105 @@ static void intel_pmu_check_extra_regs(struct extra_reg *extra_regs)
} }
} }
static void intel_pmu_check_hybrid_pmus(u64 fixed_mask) static const struct { enum hybrid_pmu_type id; char *name; } intel_hybrid_pmu_type_map[] __initconst = {
{ hybrid_small, "cpu_atom" },
{ hybrid_big, "cpu_core" },
};
static __always_inline int intel_pmu_init_hybrid(enum hybrid_pmu_type pmus)
{ {
unsigned long pmus_mask = pmus;
struct x86_hybrid_pmu *pmu; struct x86_hybrid_pmu *pmu;
int i; int idx = 0, bit;
for (i = 0; i < x86_pmu.num_hybrid_pmus; i++) { x86_pmu.num_hybrid_pmus = hweight_long(pmus_mask);
pmu = &x86_pmu.hybrid_pmu[i]; x86_pmu.hybrid_pmu = kcalloc(x86_pmu.num_hybrid_pmus,
sizeof(struct x86_hybrid_pmu),
GFP_KERNEL);
if (!x86_pmu.hybrid_pmu)
return -ENOMEM;
static_branch_enable(&perf_is_hybrid);
x86_pmu.filter = intel_pmu_filter;
intel_pmu_check_num_counters(&pmu->num_counters, for_each_set_bit(bit, &pmus_mask, ARRAY_SIZE(intel_hybrid_pmu_type_map)) {
&pmu->num_counters_fixed, pmu = &x86_pmu.hybrid_pmu[idx++];
&pmu->intel_ctrl, pmu->pmu_type = intel_hybrid_pmu_type_map[bit].id;
fixed_mask); pmu->name = intel_hybrid_pmu_type_map[bit].name;
if (pmu->intel_cap.perf_metrics) { pmu->num_counters = x86_pmu.num_counters;
pmu->intel_ctrl |= 1ULL << GLOBAL_CTRL_EN_PERF_METRICS; pmu->num_counters_fixed = x86_pmu.num_counters_fixed;
pmu->intel_ctrl |= INTEL_PMC_MSK_FIXED_SLOTS; pmu->max_pebs_events = min_t(unsigned, MAX_PEBS_EVENTS, pmu->num_counters);
pmu->unconstrained = (struct event_constraint)
__EVENT_CONSTRAINT(0, (1ULL << pmu->num_counters) - 1,
0, pmu->num_counters, 0, 0);
pmu->intel_cap.capabilities = x86_pmu.intel_cap.capabilities;
if (pmu->pmu_type & hybrid_small) {
pmu->intel_cap.perf_metrics = 0;
pmu->intel_cap.pebs_output_pt_available = 1;
pmu->mid_ack = true;
} else if (pmu->pmu_type & hybrid_big) {
pmu->intel_cap.perf_metrics = 1;
pmu->intel_cap.pebs_output_pt_available = 0;
pmu->late_ack = true;
}
} }
if (pmu->intel_cap.pebs_output_pt_available) return 0;
pmu->pmu.capabilities |= PERF_PMU_CAP_AUX_OUTPUT; }
intel_pmu_check_event_constraints(pmu->event_constraints, static __always_inline void intel_pmu_ref_cycles_ext(void)
pmu->num_counters, {
pmu->num_counters_fixed, if (!(x86_pmu.events_maskl & (INTEL_PMC_MSK_FIXED_REF_CYCLES >> INTEL_PMC_IDX_FIXED)))
pmu->intel_ctrl); intel_perfmon_event_map[PERF_COUNT_HW_REF_CPU_CYCLES] = 0x013c;
}
intel_pmu_check_extra_regs(pmu->extra_regs); static __always_inline void intel_pmu_init_glc(struct pmu *pmu)
} {
x86_pmu.late_ack = true;
x86_pmu.limit_period = glc_limit_period;
x86_pmu.pebs_aliases = NULL;
x86_pmu.pebs_prec_dist = true;
x86_pmu.pebs_block = true;
x86_pmu.flags |= PMU_FL_HAS_RSP_1;
x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
x86_pmu.flags |= PMU_FL_INSTR_LATENCY;
x86_pmu.rtm_abort_event = X86_CONFIG(.event=0xc9, .umask=0x04);
x86_pmu.lbr_pt_coexist = true;
x86_pmu.num_topdown_events = 8;
static_call_update(intel_pmu_update_topdown_event,
&icl_update_topdown_event);
static_call_update(intel_pmu_set_topdown_event_period,
&icl_set_topdown_event_period);
memcpy(hybrid_var(pmu, hw_cache_event_ids), glc_hw_cache_event_ids, sizeof(hw_cache_event_ids));
memcpy(hybrid_var(pmu, hw_cache_extra_regs), glc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
hybrid(pmu, event_constraints) = intel_glc_event_constraints;
hybrid(pmu, pebs_constraints) = intel_glc_pebs_event_constraints;
intel_pmu_ref_cycles_ext();
} }
static __always_inline bool is_mtl(u8 x86_model) static __always_inline void intel_pmu_init_grt(struct pmu *pmu)
{ {
return (x86_model == INTEL_FAM6_METEORLAKE) || x86_pmu.mid_ack = true;
(x86_model == INTEL_FAM6_METEORLAKE_L); x86_pmu.limit_period = glc_limit_period;
x86_pmu.pebs_aliases = NULL;
x86_pmu.pebs_prec_dist = true;
x86_pmu.pebs_block = true;
x86_pmu.lbr_pt_coexist = true;
x86_pmu.flags |= PMU_FL_HAS_RSP_1;
x86_pmu.flags |= PMU_FL_INSTR_LATENCY;
memcpy(hybrid_var(pmu, hw_cache_event_ids), glp_hw_cache_event_ids, sizeof(hw_cache_event_ids));
memcpy(hybrid_var(pmu, hw_cache_extra_regs), tnt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
hybrid_var(pmu, hw_cache_event_ids)[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1;
hybrid(pmu, event_constraints) = intel_grt_event_constraints;
hybrid(pmu, pebs_constraints) = intel_grt_pebs_event_constraints;
hybrid(pmu, extra_regs) = intel_grt_extra_regs;
intel_pmu_ref_cycles_ext();
} }
__init int intel_pmu_init(void) __init int intel_pmu_init(void)
...@@ -6194,28 +6323,10 @@ __init int intel_pmu_init(void) ...@@ -6194,28 +6323,10 @@ __init int intel_pmu_init(void)
break; break;
case INTEL_FAM6_ATOM_GRACEMONT: case INTEL_FAM6_ATOM_GRACEMONT:
x86_pmu.mid_ack = true; intel_pmu_init_grt(NULL);
memcpy(hw_cache_event_ids, glp_hw_cache_event_ids,
sizeof(hw_cache_event_ids));
memcpy(hw_cache_extra_regs, tnt_hw_cache_extra_regs,
sizeof(hw_cache_extra_regs));
hw_cache_event_ids[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1;
x86_pmu.event_constraints = intel_slm_event_constraints;
x86_pmu.pebs_constraints = intel_grt_pebs_event_constraints;
x86_pmu.extra_regs = intel_grt_extra_regs;
x86_pmu.pebs_aliases = NULL;
x86_pmu.pebs_prec_dist = true;
x86_pmu.pebs_block = true;
x86_pmu.lbr_pt_coexist = true;
x86_pmu.flags |= PMU_FL_HAS_RSP_1;
x86_pmu.flags |= PMU_FL_INSTR_LATENCY;
intel_pmu_pebs_data_source_grt(); intel_pmu_pebs_data_source_grt();
x86_pmu.pebs_latency_data = adl_latency_data_small; x86_pmu.pebs_latency_data = adl_latency_data_small;
x86_pmu.get_event_constraints = tnt_get_event_constraints; x86_pmu.get_event_constraints = tnt_get_event_constraints;
x86_pmu.limit_period = spr_limit_period;
td_attr = tnt_events_attrs; td_attr = tnt_events_attrs;
mem_attr = grt_mem_attrs; mem_attr = grt_mem_attrs;
extra_attr = nhm_format_attr; extra_attr = nhm_format_attr;
...@@ -6225,28 +6336,11 @@ __init int intel_pmu_init(void) ...@@ -6225,28 +6336,11 @@ __init int intel_pmu_init(void)
case INTEL_FAM6_ATOM_CRESTMONT: case INTEL_FAM6_ATOM_CRESTMONT:
case INTEL_FAM6_ATOM_CRESTMONT_X: case INTEL_FAM6_ATOM_CRESTMONT_X:
x86_pmu.mid_ack = true; intel_pmu_init_grt(NULL);
memcpy(hw_cache_event_ids, glp_hw_cache_event_ids,
sizeof(hw_cache_event_ids));
memcpy(hw_cache_extra_regs, tnt_hw_cache_extra_regs,
sizeof(hw_cache_extra_regs));
hw_cache_event_ids[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1;
x86_pmu.event_constraints = intel_slm_event_constraints;
x86_pmu.pebs_constraints = intel_grt_pebs_event_constraints;
x86_pmu.extra_regs = intel_cmt_extra_regs; x86_pmu.extra_regs = intel_cmt_extra_regs;
x86_pmu.pebs_aliases = NULL;
x86_pmu.pebs_prec_dist = true;
x86_pmu.lbr_pt_coexist = true;
x86_pmu.pebs_block = true;
x86_pmu.flags |= PMU_FL_HAS_RSP_1;
x86_pmu.flags |= PMU_FL_INSTR_LATENCY;
intel_pmu_pebs_data_source_cmt(); intel_pmu_pebs_data_source_cmt();
x86_pmu.pebs_latency_data = mtl_latency_data_small; x86_pmu.pebs_latency_data = mtl_latency_data_small;
x86_pmu.get_event_constraints = cmt_get_event_constraints; x86_pmu.get_event_constraints = cmt_get_event_constraints;
x86_pmu.limit_period = spr_limit_period;
td_attr = cmt_events_attrs; td_attr = cmt_events_attrs;
mem_attr = grt_mem_attrs; mem_attr = grt_mem_attrs;
extra_attr = cmt_format_attr; extra_attr = cmt_format_attr;
...@@ -6563,44 +6657,23 @@ __init int intel_pmu_init(void) ...@@ -6563,44 +6657,23 @@ __init int intel_pmu_init(void)
case INTEL_FAM6_SAPPHIRERAPIDS_X: case INTEL_FAM6_SAPPHIRERAPIDS_X:
case INTEL_FAM6_EMERALDRAPIDS_X: case INTEL_FAM6_EMERALDRAPIDS_X:
x86_pmu.flags |= PMU_FL_MEM_LOADS_AUX; x86_pmu.flags |= PMU_FL_MEM_LOADS_AUX;
x86_pmu.extra_regs = intel_spr_extra_regs; x86_pmu.extra_regs = intel_glc_extra_regs;
fallthrough; fallthrough;
case INTEL_FAM6_GRANITERAPIDS_X: case INTEL_FAM6_GRANITERAPIDS_X:
case INTEL_FAM6_GRANITERAPIDS_D: case INTEL_FAM6_GRANITERAPIDS_D:
pmem = true; intel_pmu_init_glc(NULL);
x86_pmu.late_ack = true;
memcpy(hw_cache_event_ids, spr_hw_cache_event_ids, sizeof(hw_cache_event_ids));
memcpy(hw_cache_extra_regs, spr_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
x86_pmu.event_constraints = intel_spr_event_constraints;
x86_pmu.pebs_constraints = intel_spr_pebs_event_constraints;
if (!x86_pmu.extra_regs) if (!x86_pmu.extra_regs)
x86_pmu.extra_regs = intel_gnr_extra_regs; x86_pmu.extra_regs = intel_rwc_extra_regs;
x86_pmu.limit_period = spr_limit_period;
x86_pmu.pebs_ept = 1; x86_pmu.pebs_ept = 1;
x86_pmu.pebs_aliases = NULL;
x86_pmu.pebs_prec_dist = true;
x86_pmu.pebs_block = true;
x86_pmu.flags |= PMU_FL_HAS_RSP_1;
x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
x86_pmu.flags |= PMU_FL_INSTR_LATENCY;
x86_pmu.hw_config = hsw_hw_config; x86_pmu.hw_config = hsw_hw_config;
x86_pmu.get_event_constraints = spr_get_event_constraints; x86_pmu.get_event_constraints = glc_get_event_constraints;
extra_attr = boot_cpu_has(X86_FEATURE_RTM) ? extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
hsw_format_attr : nhm_format_attr; hsw_format_attr : nhm_format_attr;
extra_skl_attr = skl_format_attr; extra_skl_attr = skl_format_attr;
mem_attr = spr_events_attrs; mem_attr = glc_events_attrs;
td_attr = spr_td_events_attrs; td_attr = glc_td_events_attrs;
tsx_attr = spr_tsx_events_attrs; tsx_attr = glc_tsx_events_attrs;
x86_pmu.rtm_abort_event = X86_CONFIG(.event=0xc9, .umask=0x04); intel_pmu_pebs_data_source_skl(true);
x86_pmu.lbr_pt_coexist = true;
intel_pmu_pebs_data_source_skl(pmem);
x86_pmu.num_topdown_events = 8;
static_call_update(intel_pmu_update_topdown_event,
&icl_update_topdown_event);
static_call_update(intel_pmu_set_topdown_event_period,
&icl_set_topdown_event_period);
pr_cont("Sapphire Rapids events, "); pr_cont("Sapphire Rapids events, ");
name = "sapphire_rapids"; name = "sapphire_rapids";
break; break;
...@@ -6610,47 +6683,17 @@ __init int intel_pmu_init(void) ...@@ -6610,47 +6683,17 @@ __init int intel_pmu_init(void)
case INTEL_FAM6_RAPTORLAKE: case INTEL_FAM6_RAPTORLAKE:
case INTEL_FAM6_RAPTORLAKE_P: case INTEL_FAM6_RAPTORLAKE_P:
case INTEL_FAM6_RAPTORLAKE_S: case INTEL_FAM6_RAPTORLAKE_S:
case INTEL_FAM6_METEORLAKE:
case INTEL_FAM6_METEORLAKE_L:
/* /*
* Alder Lake has 2 types of CPU, core and atom. * Alder Lake has 2 types of CPU, core and atom.
* *
* Initialize the common PerfMon capabilities here. * Initialize the common PerfMon capabilities here.
*/ */
x86_pmu.hybrid_pmu = kcalloc(X86_HYBRID_NUM_PMUS, intel_pmu_init_hybrid(hybrid_big_small);
sizeof(struct x86_hybrid_pmu),
GFP_KERNEL);
if (!x86_pmu.hybrid_pmu)
return -ENOMEM;
static_branch_enable(&perf_is_hybrid);
x86_pmu.num_hybrid_pmus = X86_HYBRID_NUM_PMUS;
x86_pmu.pebs_aliases = NULL;
x86_pmu.pebs_prec_dist = true;
x86_pmu.pebs_block = true;
x86_pmu.flags |= PMU_FL_HAS_RSP_1;
x86_pmu.flags |= PMU_FL_NO_HT_SHARING;
x86_pmu.flags |= PMU_FL_INSTR_LATENCY;
x86_pmu.lbr_pt_coexist = true;
x86_pmu.pebs_latency_data = adl_latency_data_small; x86_pmu.pebs_latency_data = adl_latency_data_small;
x86_pmu.num_topdown_events = 8;
static_call_update(intel_pmu_update_topdown_event,
&adl_update_topdown_event);
static_call_update(intel_pmu_set_topdown_event_period,
&adl_set_topdown_event_period);
x86_pmu.filter = intel_pmu_filter;
x86_pmu.get_event_constraints = adl_get_event_constraints; x86_pmu.get_event_constraints = adl_get_event_constraints;
x86_pmu.hw_config = adl_hw_config; x86_pmu.hw_config = adl_hw_config;
x86_pmu.limit_period = spr_limit_period;
x86_pmu.get_hybrid_cpu_type = adl_get_hybrid_cpu_type; x86_pmu.get_hybrid_cpu_type = adl_get_hybrid_cpu_type;
/*
* The rtm_abort_event is used to check whether to enable GPRs
* for the RTM abort event. Atom doesn't have the RTM abort
* event. There is no harmful to set it in the common
* x86_pmu.rtm_abort_event.
*/
x86_pmu.rtm_abort_event = X86_CONFIG(.event=0xc9, .umask=0x04);
td_attr = adl_hybrid_events_attrs; td_attr = adl_hybrid_events_attrs;
mem_attr = adl_hybrid_mem_attrs; mem_attr = adl_hybrid_mem_attrs;
...@@ -6660,9 +6703,7 @@ __init int intel_pmu_init(void) ...@@ -6660,9 +6703,7 @@ __init int intel_pmu_init(void)
/* Initialize big core specific PerfMon capabilities.*/ /* Initialize big core specific PerfMon capabilities.*/
pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
pmu->name = "cpu_core"; intel_pmu_init_glc(&pmu->pmu);
pmu->cpu_type = hybrid_big;
pmu->late_ack = true;
if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) { if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) {
pmu->num_counters = x86_pmu.num_counters + 2; pmu->num_counters = x86_pmu.num_counters + 2;
pmu->num_counters_fixed = x86_pmu.num_counters_fixed + 1; pmu->num_counters_fixed = x86_pmu.num_counters_fixed + 1;
...@@ -6687,54 +6728,45 @@ __init int intel_pmu_init(void) ...@@ -6687,54 +6728,45 @@ __init int intel_pmu_init(void)
pmu->unconstrained = (struct event_constraint) pmu->unconstrained = (struct event_constraint)
__EVENT_CONSTRAINT(0, (1ULL << pmu->num_counters) - 1, __EVENT_CONSTRAINT(0, (1ULL << pmu->num_counters) - 1,
0, pmu->num_counters, 0, 0); 0, pmu->num_counters, 0, 0);
pmu->intel_cap.capabilities = x86_pmu.intel_cap.capabilities; pmu->extra_regs = intel_glc_extra_regs;
pmu->intel_cap.perf_metrics = 1;
pmu->intel_cap.pebs_output_pt_available = 0;
memcpy(pmu->hw_cache_event_ids, spr_hw_cache_event_ids, sizeof(pmu->hw_cache_event_ids));
memcpy(pmu->hw_cache_extra_regs, spr_hw_cache_extra_regs, sizeof(pmu->hw_cache_extra_regs));
pmu->event_constraints = intel_spr_event_constraints;
pmu->pebs_constraints = intel_spr_pebs_event_constraints;
pmu->extra_regs = intel_spr_extra_regs;
/* Initialize Atom core specific PerfMon capabilities.*/ /* Initialize Atom core specific PerfMon capabilities.*/
pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX]; pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
pmu->name = "cpu_atom"; intel_pmu_init_grt(&pmu->pmu);
pmu->cpu_type = hybrid_small;
pmu->mid_ack = true; x86_pmu.flags |= PMU_FL_MEM_LOADS_AUX;
pmu->num_counters = x86_pmu.num_counters; intel_pmu_pebs_data_source_adl();
pmu->num_counters_fixed = x86_pmu.num_counters_fixed; pr_cont("Alderlake Hybrid events, ");
pmu->max_pebs_events = x86_pmu.max_pebs_events; name = "alderlake_hybrid";
pmu->unconstrained = (struct event_constraint) break;
__EVENT_CONSTRAINT(0, (1ULL << pmu->num_counters) - 1,
0, pmu->num_counters, 0, 0); case INTEL_FAM6_METEORLAKE:
pmu->intel_cap.capabilities = x86_pmu.intel_cap.capabilities; case INTEL_FAM6_METEORLAKE_L:
pmu->intel_cap.perf_metrics = 0; intel_pmu_init_hybrid(hybrid_big_small);
pmu->intel_cap.pebs_output_pt_available = 1;
memcpy(pmu->hw_cache_event_ids, glp_hw_cache_event_ids, sizeof(pmu->hw_cache_event_ids));
memcpy(pmu->hw_cache_extra_regs, tnt_hw_cache_extra_regs, sizeof(pmu->hw_cache_extra_regs));
pmu->hw_cache_event_ids[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1;
pmu->event_constraints = intel_slm_event_constraints;
pmu->pebs_constraints = intel_grt_pebs_event_constraints;
pmu->extra_regs = intel_grt_extra_regs;
if (is_mtl(boot_cpu_data.x86_model)) {
x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX].extra_regs = intel_gnr_extra_regs;
x86_pmu.pebs_latency_data = mtl_latency_data_small; x86_pmu.pebs_latency_data = mtl_latency_data_small;
x86_pmu.get_event_constraints = mtl_get_event_constraints;
x86_pmu.hw_config = adl_hw_config;
td_attr = adl_hybrid_events_attrs;
mem_attr = mtl_hybrid_mem_attrs;
tsx_attr = adl_hybrid_tsx_attrs;
extra_attr = boot_cpu_has(X86_FEATURE_RTM) ? extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
mtl_hybrid_extra_attr_rtm : mtl_hybrid_extra_attr; mtl_hybrid_extra_attr_rtm : mtl_hybrid_extra_attr;
mem_attr = mtl_hybrid_mem_attrs;
intel_pmu_pebs_data_source_mtl(); /* Initialize big core specific PerfMon capabilities.*/
x86_pmu.get_event_constraints = mtl_get_event_constraints; pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
intel_pmu_init_glc(&pmu->pmu);
pmu->extra_regs = intel_rwc_extra_regs;
/* Initialize Atom core specific PerfMon capabilities.*/
pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
intel_pmu_init_grt(&pmu->pmu);
pmu->extra_regs = intel_cmt_extra_regs; pmu->extra_regs = intel_cmt_extra_regs;
intel_pmu_pebs_data_source_mtl();
pr_cont("Meteorlake Hybrid events, "); pr_cont("Meteorlake Hybrid events, ");
name = "meteorlake_hybrid"; name = "meteorlake_hybrid";
} else {
x86_pmu.flags |= PMU_FL_MEM_LOADS_AUX;
intel_pmu_pebs_data_source_adl();
pr_cont("Alderlake Hybrid events, ");
name = "alderlake_hybrid";
}
break; break;
default: default:
...@@ -6846,9 +6878,6 @@ __init int intel_pmu_init(void) ...@@ -6846,9 +6878,6 @@ __init int intel_pmu_init(void)
if (!is_hybrid() && x86_pmu.intel_cap.perf_metrics) if (!is_hybrid() && x86_pmu.intel_cap.perf_metrics)
x86_pmu.intel_ctrl |= 1ULL << GLOBAL_CTRL_EN_PERF_METRICS; x86_pmu.intel_ctrl |= 1ULL << GLOBAL_CTRL_EN_PERF_METRICS;
if (is_hybrid())
intel_pmu_check_hybrid_pmus((u64)fixed_mask);
if (x86_pmu.intel_cap.pebs_timing_info) if (x86_pmu.intel_cap.pebs_timing_info)
x86_pmu.flags |= PMU_FL_RETIRE_LATENCY; x86_pmu.flags |= PMU_FL_RETIRE_LATENCY;
......
arch/x86/events/intel/cstate.c
View file @ bceb7acc
...@@ -336,6 +336,9 @@ static int cstate_pmu_event_init(struct perf_event *event) ...@@ -336,6 +336,9 @@ static int cstate_pmu_event_init(struct perf_event *event)
cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_PKG_EVENT_MAX); cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_PKG_EVENT_MAX);
if (!(pkg_msr_mask & (1 << cfg))) if (!(pkg_msr_mask & (1 << cfg)))
return -EINVAL; return -EINVAL;
event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG;
event->hw.event_base = pkg_msr[cfg].msr; event->hw.event_base = pkg_msr[cfg].msr;
cpu = cpumask_any_and(&cstate_pkg_cpu_mask, cpu = cpumask_any_and(&cstate_pkg_cpu_mask,
topology_die_cpumask(event->cpu)); topology_die_cpumask(event->cpu));
......
arch/x86/events/intel/ds.c
View file @ bceb7acc
...@@ -261,7 +261,7 @@ static u64 __adl_latency_data_small(struct perf_event *event, u64 status, ...@@ -261,7 +261,7 @@ static u64 __adl_latency_data_small(struct perf_event *event, u64 status,
{ {
u64 val; u64 val;
WARN_ON_ONCE(hybrid_pmu(event->pmu)->cpu_type == hybrid_big); WARN_ON_ONCE(hybrid_pmu(event->pmu)->pmu_type == hybrid_big);
dse &= PERF_PEBS_DATA_SOURCE_MASK; dse &= PERF_PEBS_DATA_SOURCE_MASK;
val = hybrid_var(event->pmu, pebs_data_source)[dse]; val = hybrid_var(event->pmu, pebs_data_source)[dse];
...@@ -1058,7 +1058,7 @@ struct event_constraint intel_icl_pebs_event_constraints[] = { ...@@ -1058,7 +1058,7 @@ struct event_constraint intel_icl_pebs_event_constraints[] = {
EVENT_CONSTRAINT_END EVENT_CONSTRAINT_END
}; };
struct event_constraint intel_spr_pebs_event_constraints[] = { struct event_constraint intel_glc_pebs_event_constraints[] = {
INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */ INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */
INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL), INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL),
......
arch/x86/events/intel/pt.c
View file @ bceb7acc
...@@ -736,6 +736,7 @@ static bool topa_table_full(struct topa *topa) ...@@ -736,6 +736,7 @@ static bool topa_table_full(struct topa *topa)
/** /**
* topa_insert_pages() - create a list of ToPA tables * topa_insert_pages() - create a list of ToPA tables
* @buf: PT buffer being initialized. * @buf: PT buffer being initialized.
* @cpu: CPU on which to allocate.
* @gfp: Allocation flags. * @gfp: Allocation flags.
* *
* This initializes a list of ToPA tables with entries from * This initializes a list of ToPA tables with entries from
...@@ -1207,8 +1208,11 @@ static void pt_buffer_fini_topa(struct pt_buffer *buf) ...@@ -1207,8 +1208,11 @@ static void pt_buffer_fini_topa(struct pt_buffer *buf)
/** /**
* pt_buffer_init_topa() - initialize ToPA table for pt buffer * pt_buffer_init_topa() - initialize ToPA table for pt buffer
* @buf: PT buffer. * @buf: PT buffer.
* @size: Total size of all regions within this ToPA. * @cpu: CPU on which to allocate.
* @nr_pages: No. of pages to allocate.
* @gfp: Allocation flags. * @gfp: Allocation flags.
*
* Return: 0 on success or error code.
*/ */
static int pt_buffer_init_topa(struct pt_buffer *buf, int cpu, static int pt_buffer_init_topa(struct pt_buffer *buf, int cpu,
unsigned long nr_pages, gfp_t gfp) unsigned long nr_pages, gfp_t gfp)
...@@ -1281,7 +1285,7 @@ static int pt_buffer_try_single(struct pt_buffer *buf, int nr_pages) ...@@ -1281,7 +1285,7 @@ static int pt_buffer_try_single(struct pt_buffer *buf, int nr_pages)
/** /**
* pt_buffer_setup_aux() - set up topa tables for a PT buffer * pt_buffer_setup_aux() - set up topa tables for a PT buffer
* @cpu: Cpu on which to allocate, -1 means current. * @event: Performance event
* @pages: Array of pointers to buffer pages passed from perf core. * @pages: Array of pointers to buffer pages passed from perf core.
* @nr_pages: Number of pages in the buffer. * @nr_pages: Number of pages in the buffer.
* @snapshot: If this is a snapshot/overwrite counter. * @snapshot: If this is a snapshot/overwrite counter.
......
arch/x86/events/perf_event.h
View file @ bceb7acc
...@@ -652,10 +652,29 @@ enum { ...@@ -652,10 +652,29 @@ enum {
#define PERF_PEBS_DATA_SOURCE_MAX 0x10 #define PERF_PEBS_DATA_SOURCE_MAX 0x10
#define PERF_PEBS_DATA_SOURCE_MASK (PERF_PEBS_DATA_SOURCE_MAX - 1) #define PERF_PEBS_DATA_SOURCE_MASK (PERF_PEBS_DATA_SOURCE_MAX - 1)
enum hybrid_cpu_type {
HYBRID_INTEL_NONE,
HYBRID_INTEL_ATOM = 0x20,
HYBRID_INTEL_CORE = 0x40,
};
enum hybrid_pmu_type {
not_hybrid,
hybrid_small = BIT(0),
hybrid_big = BIT(1),
hybrid_big_small = hybrid_big | hybrid_small, /* only used for matching */
};
#define X86_HYBRID_PMU_ATOM_IDX 0
#define X86_HYBRID_PMU_CORE_IDX 1
#define X86_HYBRID_NUM_PMUS 2
struct x86_hybrid_pmu { struct x86_hybrid_pmu {
struct pmu pmu; struct pmu pmu;
const char *name; const char *name;
u8 cpu_type; enum hybrid_pmu_type pmu_type;
cpumask_t supported_cpus; cpumask_t supported_cpus;
union perf_capabilities intel_cap; union perf_capabilities intel_cap;
u64 intel_ctrl; u64 intel_ctrl;
...@@ -721,18 +740,6 @@ extern struct static_key_false perf_is_hybrid; ...@@ -721,18 +740,6 @@ extern struct static_key_false perf_is_hybrid;
__Fp; \ __Fp; \
}) })
enum hybrid_pmu_type {
hybrid_big = 0x40,
hybrid_small = 0x20,
hybrid_big_small = hybrid_big | hybrid_small,
};
#define X86_HYBRID_PMU_ATOM_IDX 0
#define X86_HYBRID_PMU_CORE_IDX 1
#define X86_HYBRID_NUM_PMUS 2
/* /*
* struct x86_pmu - generic x86 pmu * struct x86_pmu - generic x86 pmu
*/ */
...@@ -940,7 +947,7 @@ struct x86_pmu { ...@@ -940,7 +947,7 @@ struct x86_pmu {
*/ */
int num_hybrid_pmus; int num_hybrid_pmus;
struct x86_hybrid_pmu *hybrid_pmu; struct x86_hybrid_pmu *hybrid_pmu;
u8 (*get_hybrid_cpu_type) (void); enum hybrid_cpu_type (*get_hybrid_cpu_type) (void);
}; };
struct x86_perf_task_context_opt { struct x86_perf_task_context_opt {
...@@ -1521,7 +1528,7 @@ extern struct event_constraint intel_skl_pebs_event_constraints[]; ...@@ -1521,7 +1528,7 @@ extern struct event_constraint intel_skl_pebs_event_constraints[];
extern struct event_constraint intel_icl_pebs_event_constraints[]; extern struct event_constraint intel_icl_pebs_event_constraints[];
extern struct event_constraint intel_spr_pebs_event_constraints[]; extern struct event_constraint intel_glc_pebs_event_constraints[];
struct event_constraint *intel_pebs_constraints(struct perf_event *event); struct event_constraint *intel_pebs_constraints(struct perf_event *event);
......
arch/x86/events/rapl.c
View file @ bceb7acc
...@@ -115,7 +115,7 @@ struct rapl_pmu { ...@@ -115,7 +115,7 @@ struct rapl_pmu {
struct rapl_pmus { struct rapl_pmus {
struct pmu pmu; struct pmu pmu;
unsigned int maxdie; unsigned int maxdie;
struct rapl_pmu *pmus[]; struct rapl_pmu *pmus[] __counted_by(maxdie);
}; };
enum rapl_unit_quirk { enum rapl_unit_quirk {
...@@ -179,15 +179,11 @@ static u64 rapl_event_update(struct perf_event *event) ...@@ -179,15 +179,11 @@ static u64 rapl_event_update(struct perf_event *event)
s64 delta, sdelta; s64 delta, sdelta;
int shift = RAPL_CNTR_WIDTH; int shift = RAPL_CNTR_WIDTH;
again:
prev_raw_count = local64_read(&hwc->prev_count); prev_raw_count = local64_read(&hwc->prev_count);
do {
rdmsrl(event->hw.event_base, new_raw_count); rdmsrl(event->hw.event_base, new_raw_count);
} while (!local64_try_cmpxchg(&hwc->prev_count,
if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, &prev_raw_count, new_raw_count));
new_raw_count) != prev_raw_count) {
cpu_relax();
goto again;
}
/* /*
* Now we have the new raw value and have updated the prev * Now we have the new raw value and have updated the prev
...@@ -537,11 +533,11 @@ static struct perf_msr intel_rapl_spr_msrs[] = { ...@@ -537,11 +533,11 @@ static struct perf_msr intel_rapl_spr_msrs[] = {
* - want to use same event codes across both architectures * - want to use same event codes across both architectures
*/ */
static struct perf_msr amd_rapl_msrs[] = { static struct perf_msr amd_rapl_msrs[] = {
[PERF_RAPL_PP0] = { 0, &rapl_events_cores_group, 0, false, 0 }, [PERF_RAPL_PP0] = { 0, &rapl_events_cores_group, NULL, false, 0 },
[PERF_RAPL_PKG] = { MSR_AMD_PKG_ENERGY_STATUS, &rapl_events_pkg_group, test_msr, false, RAPL_MSR_MASK }, [PERF_RAPL_PKG] = { MSR_AMD_PKG_ENERGY_STATUS, &rapl_events_pkg_group, test_msr, false, RAPL_MSR_MASK },
[PERF_RAPL_RAM] = { 0, &rapl_events_ram_group, 0, false, 0 }, [PERF_RAPL_RAM] = { 0, &rapl_events_ram_group, NULL, false, 0 },
[PERF_RAPL_PP1] = { 0, &rapl_events_gpu_group, 0, false, 0 }, [PERF_RAPL_PP1] = { 0, &rapl_events_gpu_group, NULL, false, 0 },
[PERF_RAPL_PSYS] = { 0, &rapl_events_psys_group, 0, false, 0 }, [PERF_RAPL_PSYS] = { 0, &rapl_events_psys_group, NULL, false, 0 },
}; };
static int rapl_cpu_offline(unsigned int cpu) static int rapl_cpu_offline(unsigned int cpu)
......
arch/x86/include/asm/msr-index.h
View file @ bceb7acc
...@@ -641,6 +641,10 @@ ...@@ -641,6 +641,10 @@
#define MSR_ZEN4_BP_CFG 0xc001102e #define MSR_ZEN4_BP_CFG 0xc001102e
#define MSR_ZEN4_BP_CFG_SHARED_BTB_FIX_BIT 5 #define MSR_ZEN4_BP_CFG_SHARED_BTB_FIX_BIT 5
/* Fam 19h MSRs */
#define MSR_F19H_UMC_PERF_CTL 0xc0010800
#define MSR_F19H_UMC_PERF_CTR 0xc0010801
/* Zen 2 */ /* Zen 2 */
#define MSR_ZEN2_SPECTRAL_CHICKEN 0xc00110e3 #define MSR_ZEN2_SPECTRAL_CHICKEN 0xc00110e3
#define MSR_ZEN2_SPECTRAL_CHICKEN_BIT BIT_ULL(1) #define MSR_ZEN2_SPECTRAL_CHICKEN_BIT BIT_ULL(1)
......
arch/x86/include/asm/perf_event.h
View file @ bceb7acc
...@@ -112,6 +112,13 @@ ...@@ -112,6 +112,13 @@
(AMD64_PERFMON_V2_EVENTSEL_EVENT_NB | \ (AMD64_PERFMON_V2_EVENTSEL_EVENT_NB | \
AMD64_PERFMON_V2_EVENTSEL_UMASK_NB) AMD64_PERFMON_V2_EVENTSEL_UMASK_NB)
#define AMD64_PERFMON_V2_ENABLE_UMC BIT_ULL(31)
#define AMD64_PERFMON_V2_EVENTSEL_EVENT_UMC GENMASK_ULL(7, 0)
#define AMD64_PERFMON_V2_EVENTSEL_RDWRMASK_UMC GENMASK_ULL(9, 8)
#define AMD64_PERFMON_V2_RAW_EVENT_MASK_UMC \
(AMD64_PERFMON_V2_EVENTSEL_EVENT_UMC | \
AMD64_PERFMON_V2_EVENTSEL_RDWRMASK_UMC)
#define AMD64_NUM_COUNTERS 4 #define AMD64_NUM_COUNTERS 4
#define AMD64_NUM_COUNTERS_CORE 6 #define AMD64_NUM_COUNTERS_CORE 6
#define AMD64_NUM_COUNTERS_NB 4 #define AMD64_NUM_COUNTERS_NB 4
...@@ -232,6 +239,8 @@ union cpuid_0x80000022_ebx { ...@@ -232,6 +239,8 @@ union cpuid_0x80000022_ebx {
unsigned int lbr_v2_stack_sz:6; unsigned int lbr_v2_stack_sz:6;
/* Number of Data Fabric Counters */ /* Number of Data Fabric Counters */
unsigned int num_df_pmc:6; unsigned int num_df_pmc:6;
/* Number of Unified Memory Controller Counters */
unsigned int num_umc_pmc:6;
} split; } split;
unsigned int full; unsigned int full;
}; };
......
include/linux/perf_event.h
View file @ bceb7acc
...@@ -879,6 +879,7 @@ struct perf_event_pmu_context { ...@@ -879,6 +879,7 @@ struct perf_event_pmu_context {
unsigned int embedded : 1; unsigned int embedded : 1;
unsigned int nr_events; unsigned int nr_events;
unsigned int nr_cgroups;
atomic_t refcount; /* event <-> epc */ atomic_t refcount; /* event <-> epc */
struct rcu_head rcu_head; struct rcu_head rcu_head;
...@@ -1574,7 +1575,7 @@ extern int sysctl_perf_cpu_time_max_percent; ...@@ -1574,7 +1575,7 @@ extern int sysctl_perf_cpu_time_max_percent;
extern void perf_sample_event_took(u64 sample_len_ns); extern void perf_sample_event_took(u64 sample_len_ns);
int perf_proc_update_handler(struct ctl_table *table, int write, int perf_event_max_sample_rate_handler(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos); void *buffer, size_t *lenp, loff_t *ppos);
int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write, int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos); void *buffer, size_t *lenp, loff_t *ppos);
......
kernel/events/core.c
View file @ bceb7acc
...@@ -375,6 +375,7 @@ enum event_type_t { ...@@ -375,6 +375,7 @@ enum event_type_t {
EVENT_TIME = 0x4, EVENT_TIME = 0x4,
/* see ctx_resched() for details */ /* see ctx_resched() for details */
EVENT_CPU = 0x8, EVENT_CPU = 0x8,
EVENT_CGROUP = 0x10,
EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED, EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
}; };
...@@ -449,7 +450,7 @@ static void update_perf_cpu_limits(void) ...@@ -449,7 +450,7 @@ static void update_perf_cpu_limits(void)
static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc); static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc);
int perf_proc_update_handler(struct ctl_table *table, int write, int perf_event_max_sample_rate_handler(struct ctl_table *table, int write,
void *buffer, size_t *lenp, loff_t *ppos) void *buffer, size_t *lenp, loff_t *ppos)
{ {
int ret; int ret;
...@@ -684,20 +685,26 @@ do { \ ...@@ -684,20 +685,26 @@ do { \
___p; \ ___p; \
}) })
static void perf_ctx_disable(struct perf_event_context *ctx) static void perf_ctx_disable(struct perf_event_context *ctx, bool cgroup)
{ {
struct perf_event_pmu_context *pmu_ctx; struct perf_event_pmu_context *pmu_ctx;
list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
if (cgroup && !pmu_ctx->nr_cgroups)
continue;
perf_pmu_disable(pmu_ctx->pmu); perf_pmu_disable(pmu_ctx->pmu);
}
} }
static void perf_ctx_enable(struct perf_event_context *ctx) static void perf_ctx_enable(struct perf_event_context *ctx, bool cgroup)
{ {
struct perf_event_pmu_context *pmu_ctx; struct perf_event_pmu_context *pmu_ctx;
list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
if (cgroup && !pmu_ctx->nr_cgroups)
continue;
perf_pmu_enable(pmu_ctx->pmu); perf_pmu_enable(pmu_ctx->pmu);
}
} }
static void ctx_sched_out(struct perf_event_context *ctx, enum event_type_t event_type); static void ctx_sched_out(struct perf_event_context *ctx, enum event_type_t event_type);
...@@ -856,9 +863,9 @@ static void perf_cgroup_switch(struct task_struct *task) ...@@ -856,9 +863,9 @@ static void perf_cgroup_switch(struct task_struct *task)
return; return;
perf_ctx_lock(cpuctx, cpuctx->task_ctx); perf_ctx_lock(cpuctx, cpuctx->task_ctx);
perf_ctx_disable(&cpuctx->ctx); perf_ctx_disable(&cpuctx->ctx, true);
ctx_sched_out(&cpuctx->ctx, EVENT_ALL); ctx_sched_out(&cpuctx->ctx, EVENT_ALL|EVENT_CGROUP);
/* /*
* must not be done before ctxswout due * must not be done before ctxswout due
* to update_cgrp_time_from_cpuctx() in * to update_cgrp_time_from_cpuctx() in
...@@ -870,9 +877,9 @@ static void perf_cgroup_switch(struct task_struct *task) ...@@ -870,9 +877,9 @@ static void perf_cgroup_switch(struct task_struct *task)
* perf_cgroup_set_timestamp() in ctx_sched_in() * perf_cgroup_set_timestamp() in ctx_sched_in()
* to not have to pass task around * to not have to pass task around
*/ */
ctx_sched_in(&cpuctx->ctx, EVENT_ALL); ctx_sched_in(&cpuctx->ctx, EVENT_ALL|EVENT_CGROUP);
perf_ctx_enable(&cpuctx->ctx); perf_ctx_enable(&cpuctx->ctx, true);
perf_ctx_unlock(cpuctx, cpuctx->task_ctx); perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
} }
...@@ -965,6 +972,8 @@ perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ct ...@@ -965,6 +972,8 @@ perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ct
if (!is_cgroup_event(event)) if (!is_cgroup_event(event))
return; return;
event->pmu_ctx->nr_cgroups++;
/* /*
* Because cgroup events are always per-cpu events, * Because cgroup events are always per-cpu events,
* @ctx == &cpuctx->ctx. * @ctx == &cpuctx->ctx.
...@@ -985,6 +994,8 @@ perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *c ...@@ -985,6 +994,8 @@ perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *c
if (!is_cgroup_event(event)) if (!is_cgroup_event(event))
return; return;
event->pmu_ctx->nr_cgroups--;
/* /*
* Because cgroup events are always per-cpu events, * Because cgroup events are always per-cpu events,
* @ctx == &cpuctx->ctx. * @ctx == &cpuctx->ctx.
...@@ -2679,9 +2690,9 @@ static void ctx_resched(struct perf_cpu_context *cpuctx, ...@@ -2679,9 +2690,9 @@ static void ctx_resched(struct perf_cpu_context *cpuctx,
event_type &= EVENT_ALL; event_type &= EVENT_ALL;
perf_ctx_disable(&cpuctx->ctx); perf_ctx_disable(&cpuctx->ctx, false);
if (task_ctx) { if (task_ctx) {
perf_ctx_disable(task_ctx); perf_ctx_disable(task_ctx, false);
task_ctx_sched_out(task_ctx, event_type); task_ctx_sched_out(task_ctx, event_type);
} }
...@@ -2699,9 +2710,9 @@ static void ctx_resched(struct perf_cpu_context *cpuctx, ...@@ -2699,9 +2710,9 @@ static void ctx_resched(struct perf_cpu_context *cpuctx,
perf_event_sched_in(cpuctx, task_ctx); perf_event_sched_in(cpuctx, task_ctx);
perf_ctx_enable(&cpuctx->ctx); perf_ctx_enable(&cpuctx->ctx, false);
if (task_ctx) if (task_ctx)
perf_ctx_enable(task_ctx); perf_ctx_enable(task_ctx, false);
} }
void perf_pmu_resched(struct pmu *pmu) void perf_pmu_resched(struct pmu *pmu)
...@@ -3246,6 +3257,9 @@ ctx_sched_out(struct perf_event_context *ctx, enum event_type_t event_type) ...@@ -3246,6 +3257,9 @@ ctx_sched_out(struct perf_event_context *ctx, enum event_type_t event_type)
struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
struct perf_event_pmu_context *pmu_ctx; struct perf_event_pmu_context *pmu_ctx;
int is_active = ctx->is_active; int is_active = ctx->is_active;
bool cgroup = event_type & EVENT_CGROUP;
event_type &= ~EVENT_CGROUP;
lockdep_assert_held(&ctx->lock); lockdep_assert_held(&ctx->lock);
...@@ -3292,8 +3306,11 @@ ctx_sched_out(struct perf_event_context *ctx, enum event_type_t event_type) ...@@ -3292,8 +3306,11 @@ ctx_sched_out(struct perf_event_context *ctx, enum event_type_t event_type)
is_active ^= ctx->is_active; /* changed bits */ is_active ^= ctx->is_active; /* changed bits */
list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
if (cgroup && !pmu_ctx->nr_cgroups)
continue;
__pmu_ctx_sched_out(pmu_ctx, is_active); __pmu_ctx_sched_out(pmu_ctx, is_active);
}
} }
/* /*
...@@ -3484,7 +3501,7 @@ perf_event_context_sched_out(struct task_struct *task, struct task_struct *next) ...@@ -3484,7 +3501,7 @@ perf_event_context_sched_out(struct task_struct *task, struct task_struct *next)
raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
if (context_equiv(ctx, next_ctx)) { if (context_equiv(ctx, next_ctx)) {
perf_ctx_disable(ctx); perf_ctx_disable(ctx, false);
/* PMIs are disabled; ctx->nr_pending is stable. */ /* PMIs are disabled; ctx->nr_pending is stable. */
if (local_read(&ctx->nr_pending) || if (local_read(&ctx->nr_pending) ||
...@@ -3504,7 +3521,7 @@ perf_event_context_sched_out(struct task_struct *task, struct task_struct *next) ...@@ -3504,7 +3521,7 @@ perf_event_context_sched_out(struct task_struct *task, struct task_struct *next)
perf_ctx_sched_task_cb(ctx, false); perf_ctx_sched_task_cb(ctx, false);
perf_event_swap_task_ctx_data(ctx, next_ctx); perf_event_swap_task_ctx_data(ctx, next_ctx);
perf_ctx_enable(ctx); perf_ctx_enable(ctx, false);
/* /*
* RCU_INIT_POINTER here is safe because we've not * RCU_INIT_POINTER here is safe because we've not
...@@ -3528,13 +3545,13 @@ perf_event_context_sched_out(struct task_struct *task, struct task_struct *next) ...@@ -3528,13 +3545,13 @@ perf_event_context_sched_out(struct task_struct *task, struct task_struct *next)
if (do_switch) { if (do_switch) {
raw_spin_lock(&ctx->lock); raw_spin_lock(&ctx->lock);
perf_ctx_disable(ctx); perf_ctx_disable(ctx, false);
inside_switch: inside_switch:
perf_ctx_sched_task_cb(ctx, false); perf_ctx_sched_task_cb(ctx, false);
task_ctx_sched_out(ctx, EVENT_ALL); task_ctx_sched_out(ctx, EVENT_ALL);
perf_ctx_enable(ctx); perf_ctx_enable(ctx, false);
raw_spin_unlock(&ctx->lock); raw_spin_unlock(&ctx->lock);
} }
} }
...@@ -3820,47 +3837,32 @@ static int merge_sched_in(struct perf_event *event, void *data) ...@@ -3820,47 +3837,32 @@ static int merge_sched_in(struct perf_event *event, void *data)
return 0; return 0;
} }
static void ctx_pinned_sched_in(struct perf_event_context *ctx, struct pmu *pmu) static void pmu_groups_sched_in(struct perf_event_context *ctx,
struct perf_event_groups *groups,
struct pmu *pmu)
{ {
struct perf_event_pmu_context *pmu_ctx;
int can_add_hw = 1; int can_add_hw = 1;
visit_groups_merge(ctx, groups, smp_processor_id(), pmu,
if (pmu) {
visit_groups_merge(ctx, &ctx->pinned_groups,
smp_processor_id(), pmu,
merge_sched_in, &can_add_hw);
} else {
list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
can_add_hw = 1;
visit_groups_merge(ctx, &ctx->pinned_groups,
smp_processor_id(), pmu_ctx->pmu,
merge_sched_in, &can_add_hw); merge_sched_in, &can_add_hw);
}
}
} }
static void ctx_flexible_sched_in(struct perf_event_context *ctx, struct pmu *pmu) static void ctx_groups_sched_in(struct perf_event_context *ctx,
struct perf_event_groups *groups,
bool cgroup)
{ {
struct perf_event_pmu_context *pmu_ctx; struct perf_event_pmu_context *pmu_ctx;
int can_add_hw = 1;
if (pmu) {
visit_groups_merge(ctx, &ctx->flexible_groups,
smp_processor_id(), pmu,
merge_sched_in, &can_add_hw);
} else {
list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) {
can_add_hw = 1; if (cgroup && !pmu_ctx->nr_cgroups)
visit_groups_merge(ctx, &ctx->flexible_groups, continue;
smp_processor_id(), pmu_ctx->pmu, pmu_groups_sched_in(ctx, groups, pmu_ctx->pmu);
merge_sched_in, &can_add_hw);
}
} }
} }
static void __pmu_ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu) static void __pmu_ctx_sched_in(struct perf_event_context *ctx,
struct pmu *pmu)
{ {
ctx_flexible_sched_in(ctx, pmu); pmu_groups_sched_in(ctx, &ctx->flexible_groups, pmu);
} }
static void static void
...@@ -3868,6 +3870,9 @@ ctx_sched_in(struct perf_event_context *ctx, enum event_type_t event_type) ...@@ -3868,6 +3870,9 @@ ctx_sched_in(struct perf_event_context *ctx, enum event_type_t event_type)
{ {
struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context);
int is_active = ctx->is_active; int is_active = ctx->is_active;
bool cgroup = event_type & EVENT_CGROUP;
event_type &= ~EVENT_CGROUP;
lockdep_assert_held(&ctx->lock); lockdep_assert_held(&ctx->lock);
...@@ -3900,11 +3905,11 @@ ctx_sched_in(struct perf_event_context *ctx, enum event_type_t event_type) ...@@ -3900,11 +3905,11 @@ ctx_sched_in(struct perf_event_context *ctx, enum event_type_t event_type)
* in order to give them the best chance of going on. * in order to give them the best chance of going on.
*/ */
if (is_active & EVENT_PINNED) if (is_active & EVENT_PINNED)
ctx_pinned_sched_in(ctx, NULL); ctx_groups_sched_in(ctx, &ctx->pinned_groups, cgroup);
/* Then walk through the lower prio flexible groups */ /* Then walk through the lower prio flexible groups */
if (is_active & EVENT_FLEXIBLE) if (is_active & EVENT_FLEXIBLE)
ctx_flexible_sched_in(ctx, NULL); ctx_groups_sched_in(ctx, &ctx->flexible_groups, cgroup);
} }
static void perf_event_context_sched_in(struct task_struct *task) static void perf_event_context_sched_in(struct task_struct *task)
...@@ -3919,11 +3924,11 @@ static void perf_event_context_sched_in(struct task_struct *task) ...@@ -3919,11 +3924,11 @@ static void perf_event_context_sched_in(struct task_struct *task)
if (cpuctx->task_ctx == ctx) { if (cpuctx->task_ctx == ctx) {
perf_ctx_lock(cpuctx, ctx); perf_ctx_lock(cpuctx, ctx);
perf_ctx_disable(ctx); perf_ctx_disable(ctx, false);
perf_ctx_sched_task_cb(ctx, true); perf_ctx_sched_task_cb(ctx, true);
perf_ctx_enable(ctx); perf_ctx_enable(ctx, false);
perf_ctx_unlock(cpuctx, ctx); perf_ctx_unlock(cpuctx, ctx);
goto rcu_unlock; goto rcu_unlock;
} }
...@@ -3936,7 +3941,7 @@ static void perf_event_context_sched_in(struct task_struct *task) ...@@ -3936,7 +3941,7 @@ static void perf_event_context_sched_in(struct task_struct *task)
if (!ctx->nr_events) if (!ctx->nr_events)
goto unlock; goto unlock;
perf_ctx_disable(ctx); perf_ctx_disable(ctx, false);
/* /*
* We want to keep the following priority order: * We want to keep the following priority order:
* cpu pinned (that don't need to move), task pinned, * cpu pinned (that don't need to move), task pinned,
...@@ -3946,7 +3951,7 @@ static void perf_event_context_sched_in(struct task_struct *task) ...@@ -3946,7 +3951,7 @@ static void perf_event_context_sched_in(struct task_struct *task)
* events, no need to flip the cpuctx's events around. * events, no need to flip the cpuctx's events around.
*/ */
if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) { if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) {
perf_ctx_disable(&cpuctx->ctx); perf_ctx_disable(&cpuctx->ctx, false);
ctx_sched_out(&cpuctx->ctx, EVENT_FLEXIBLE); ctx_sched_out(&cpuctx->ctx, EVENT_FLEXIBLE);
} }
...@@ -3955,9 +3960,9 @@ static void perf_event_context_sched_in(struct task_struct *task) ...@@ -3955,9 +3960,9 @@ static void perf_event_context_sched_in(struct task_struct *task)
perf_ctx_sched_task_cb(cpuctx->task_ctx, true); perf_ctx_sched_task_cb(cpuctx->task_ctx, true);
if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree))
perf_ctx_enable(&cpuctx->ctx); perf_ctx_enable(&cpuctx->ctx, false);
perf_ctx_enable(ctx); perf_ctx_enable(ctx, false);
unlock: unlock:
perf_ctx_unlock(cpuctx, ctx); perf_ctx_unlock(cpuctx, ctx);
...@@ -4427,6 +4432,9 @@ static int __perf_event_read_cpu(struct perf_event *event, int event_cpu) ...@@ -4427,6 +4432,9 @@ static int __perf_event_read_cpu(struct perf_event *event, int event_cpu)
{ {
u16 local_pkg, event_pkg; u16 local_pkg, event_pkg;
if ((unsigned)event_cpu >= nr_cpu_ids)
return event_cpu;
if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) { if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) {
int local_cpu = smp_processor_id(); int local_cpu = smp_processor_id();
...@@ -4529,6 +4537,8 @@ int perf_event_read_local(struct perf_event *event, u64 *value, ...@@ -4529,6 +4537,8 @@ int perf_event_read_local(struct perf_event *event, u64 *value,
u64 *enabled, u64 *running) u64 *enabled, u64 *running)
{ {
unsigned long flags; unsigned long flags;
int event_oncpu;
int event_cpu;
int ret = 0; int ret = 0;
/* /*
...@@ -4553,15 +4563,22 @@ int perf_event_read_local(struct perf_event *event, u64 *value, ...@@ -4553,15 +4563,22 @@ int perf_event_read_local(struct perf_event *event, u64 *value,
goto out; goto out;
} }
/*
* Get the event CPU numbers, and adjust them to local if the event is
* a per-package event that can be read locally
*/
event_oncpu = __perf_event_read_cpu(event, event->oncpu);
event_cpu = __perf_event_read_cpu(event, event->cpu);
/* If this is a per-CPU event, it must be for this CPU */ /* If this is a per-CPU event, it must be for this CPU */
if (!(event->attach_state & PERF_ATTACH_TASK) && if (!(event->attach_state & PERF_ATTACH_TASK) &&
event->cpu != smp_processor_id()) { event_cpu != smp_processor_id()) {
ret = -EINVAL; ret = -EINVAL;
goto out; goto out;
} }
/* If this is a pinned event it must be running on this CPU */ /* If this is a pinned event it must be running on this CPU */
if (event->attr.pinned && event->oncpu != smp_processor_id()) { if (event->attr.pinned && event_oncpu != smp_processor_id()) {
ret = -EBUSY; ret = -EBUSY;
goto out; goto out;
} }
...@@ -4571,7 +4588,7 @@ int perf_event_read_local(struct perf_event *event, u64 *value, ...@@ -4571,7 +4588,7 @@ int perf_event_read_local(struct perf_event *event, u64 *value,
* or local to this CPU. Furthermore it means its ACTIVE (otherwise * or local to this CPU. Furthermore it means its ACTIVE (otherwise
* oncpu == -1). * oncpu == -1).
*/ */
if (event->oncpu == smp_processor_id()) if (event_oncpu == smp_processor_id())
event->pmu->read(event); event->pmu->read(event);
*value = local64_read(&event->count); *value = local64_read(&event->count);
......
kernel/events/ring_buffer.c
View file @ bceb7acc
...@@ -700,6 +700,12 @@ int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event, ...@@ -700,6 +700,12 @@ int rb_alloc_aux(struct perf_buffer *rb, struct perf_event *event,
watermark = 0; watermark = 0;
} }
/*
* kcalloc_node() is unable to allocate buffer if the size is larger
* than: PAGE_SIZE << MAX_ORDER; directly bail out in this case.
*/
if (get_order((unsigned long)nr_pages * sizeof(void *)) > MAX_ORDER)
return -ENOMEM;
rb->aux_pages = kcalloc_node(nr_pages, sizeof(void *), GFP_KERNEL, rb->aux_pages = kcalloc_node(nr_pages, sizeof(void *), GFP_KERNEL,
node); node);
if (!rb->aux_pages) if (!rb->aux_pages)
......
kernel/sysctl.c
View file @ bceb7acc
...@@ -1983,7 +1983,7 @@ static struct ctl_table kern_table[] = { ...@@ -1983,7 +1983,7 @@ static struct ctl_table kern_table[] = {
.data = &sysctl_perf_event_sample_rate, .data = &sysctl_perf_event_sample_rate,
.maxlen = sizeof(sysctl_perf_event_sample_rate), .maxlen = sizeof(sysctl_perf_event_sample_rate),
.mode = 0644, .mode = 0644,
.proc_handler = perf_proc_update_handler, .proc_handler = perf_event_max_sample_rate_handler,
.extra1 = SYSCTL_ONE, .extra1 = SYSCTL_ONE,
}, },
{ {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7