diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index b1eeb8a751b1..4f3baecccb0b 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -646,6 +646,7 @@ struct perf_event { int oncpu; int cpu; + cpumask_t readable_on_cpus; struct list_head owner_entry; struct task_struct *owner; diff --git a/kernel/events/core.c b/kernel/events/core.c index 1ed727c799b4..6693615e7698 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -3901,10 +3901,12 @@ struct perf_read_data { static int __perf_event_read_cpu(struct perf_event *event, int event_cpu) { u16 local_pkg, event_pkg; + int local_cpu = smp_processor_id(); + + if (cpumask_test_cpu(local_cpu, &event->readable_on_cpus)) + return local_cpu; if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) { - int local_cpu = smp_processor_id(); - event_pkg = topology_physical_package_id(event_cpu); local_pkg = topology_physical_package_id(local_cpu); @@ -3996,7 +3998,8 @@ int perf_event_read_local(struct perf_event *event, u64 *value, { unsigned long flags; int ret = 0; - + int local_cpu = smp_processor_id(); + bool readable = cpumask_test_cpu(local_cpu, &event->readable_on_cpus); /* * Disabling interrupts avoids all counter scheduling (context * switches, timer based rotation and IPIs). @@ -4021,7 +4024,8 @@ int perf_event_read_local(struct perf_event *event, u64 *value, /* If this is a per-CPU event, it must be for this CPU */ if (!(event->attach_state & PERF_ATTACH_TASK) && - event->cpu != smp_processor_id()) { + event->cpu != local_cpu && + !readable) { ret = -EINVAL; goto out; } @@ -4037,7 +4041,7 @@ int perf_event_read_local(struct perf_event *event, u64 *value, * or local to this CPU. Furthermore it means its ACTIVE (otherwise * oncpu == -1). */ - if (event->oncpu == smp_processor_id()) + if (event->oncpu == smp_processor_id() || readable) event->pmu->read(event); *value = local64_read(&event->count); @@ -4062,11 +4066,13 @@ static int perf_event_read(struct perf_event *event, bool group) enum perf_event_state state = READ_ONCE(event->state); int event_cpu, ret = 0; bool active_event_skip_read = false; + bool readable; /* * If event is enabled and currently active on a CPU, update the * value in the event structure: */ + preempt_disable(); again: if (state == PERF_EVENT_STATE_ACTIVE) { @@ -4077,13 +4083,16 @@ again: * Matches the smp_wmb() from event_sched_in(). */ smp_rmb(); - event_cpu = READ_ONCE(event->oncpu); - if ((unsigned)event_cpu >= nr_cpu_ids) + readable = cpumask_test_cpu(smp_processor_id(), + &event->readable_on_cpus); + if ((unsigned int)event_cpu >= nr_cpu_ids) { + preempt_enable(); return 0; + } if (cpu_isolated(event_cpu) || (event->attr.exclude_idle && - per_cpu(is_idle, event_cpu)) || + per_cpu(is_idle, event_cpu) && !readable) || per_cpu(is_hotplugging, event_cpu)) active_event_skip_read = true; } @@ -4095,7 +4104,6 @@ again: .ret = 0, }; - preempt_disable(); event_cpu = __perf_event_read_cpu(event, event_cpu); /* @@ -4110,7 +4118,6 @@ again: */ (void)smp_call_function_single(event_cpu, __perf_event_read, &data, 1); - preempt_enable(); ret = data.ret; } else if (state == PERF_EVENT_STATE_INACTIVE || (active_event_skip_read && @@ -4139,6 +4146,8 @@ again: raw_spin_unlock_irqrestore(&ctx->lock, flags); } + preempt_enable(); + return ret; }