Commit b18d6289 authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'x86-apic-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 APIC updates from Thomas Gleixner:
 "This update provides a major overhaul of the APIC initialization and
  vector allocation code:

   - Unification of the APIC and interrupt mode setup which was
     scattered all over the place and was hard to follow. This also
     distangles the timer setup from the APIC initialization which
     brings a clear separation of functionality.

     Great detective work from Dou Lyiang!

   - Refactoring of the x86 vector allocation mechanism. The existing
     code was based on nested loops and rather convoluted APIC callbacks
     which had a horrible worst case behaviour and tried to serve all
     different use cases in one go. This led to quite odd hacks when
     supporting the new managed interupt facility for multiqueue devices
     and made it more or less impossible to deal with the vector space
     exhaustion which was a major roadblock for server hibernation.

     Aside of that the code dealing with cpu hotplug and the system
     vectors was disconnected from the actual vector management and
     allocation code, which made it hard to follow and maintain.

     Utilizing the new bitmap matrix allocator core mechanism, the new
     allocator and management code consolidates the handling of system
     vectors, legacy vectors, cpu hotplug mechanisms and the actual
     allocation which needs to be aware of system and legacy vectors and
     hotplug constraints into a single consistent entity.

     This has one visible change: The support for multi CPU targets of
     interrupts, which is only available on a certain subset of
     CPUs/APIC variants has been removed in favour of single interrupt
     targets. A proper analysis of the multi CPU target feature revealed
     that there is no real advantage as the vast majority of interrupts
     end up on the CPU with the lowest APIC id in the set of target CPUs
     anyway. That change was agreed on by the relevant folks and allowed
     to simplify the implementation significantly and to replace rather
     fragile constructs like the vector cleanup IPI with straight
     forward and solid code.

     Furthermore this allowed to cleanly separate the allocation details
     for legacy, normal and managed interrupts:

      * Legacy interrupts are not longer wasting 16 vectors
        unconditionally

      * Managed interrupts have now a guaranteed vector reservation, but
        the actual vector assignment happens when the interrupt is
        requested. It's guaranteed not to fail.

      * Normal interrupts no longer allocate vectors unconditionally
        when the interrupt is set up (IO/APIC init or MSI(X) enable).
        The mechanism has been switched to a best effort reservation
        mode. The actual allocation happens when the interrupt is
        requested. Contrary to managed interrupts the request can fail
        due to vector space exhaustion, but drivers must handle a fail
        of request_irq() anyway. When the interrupt is freed, the vector
        is handed back as well.

        This solves a long standing problem with large unconditional
        vector allocations for a certain class of enterprise devices
        which prevented server hibernation due to vector space
        exhaustion when the unused allocated vectors had to be migrated
        to CPU0 while unplugging all non boot CPUs.

     The code has been equipped with trace points and detailed debugfs
     information to aid analysis of the vector space"

* 'x86-apic-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (60 commits)
  x86/vector/msi: Select CONFIG_GENERIC_IRQ_RESERVATION_MODE
  PCI/MSI: Set MSI_FLAG_MUST_REACTIVATE in core code
  genirq: Add config option for reservation mode
  x86/vector: Use correct per cpu variable in free_moved_vector()
  x86/apic/vector: Ignore set_affinity call for inactive interrupts
  x86/apic: Fix spelling mistake: "symmectic" -> "symmetric"
  x86/apic: Use dead_cpu instead of current CPU when cleaning up
  ACPI/init: Invoke early ACPI initialization earlier
  x86/vector: Respect affinity mask in irq descriptor
  x86/irq: Simplify hotplug vector accounting
  x86/vector: Switch IOAPIC to global reservation mode
  x86/vector/msi: Switch to global reservation mode
  x86/vector: Handle managed interrupts proper
  x86/io_apic: Reevaluate vector configuration on activate()
  iommu/amd: Reevaluate vector configuration on activate()
  iommu/vt-d: Reevaluate vector configuration on activate()
  x86/apic/msi: Force reactivation of interrupts at startup time
  x86/vector: Untangle internal state from irq_cfg
  x86/vector: Compile SMP only code conditionally
  x86/apic: Remove unused callbacks
  ...
parents 7d58e1c9 141d3b1d
...@@ -93,8 +93,10 @@ config X86 ...@@ -93,8 +93,10 @@ config X86
select GENERIC_FIND_FIRST_BIT select GENERIC_FIND_FIRST_BIT
select GENERIC_IOMAP select GENERIC_IOMAP
select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP
select GENERIC_IRQ_MATRIX_ALLOCATOR if X86_LOCAL_APIC
select GENERIC_IRQ_MIGRATION if SMP select GENERIC_IRQ_MIGRATION if SMP
select GENERIC_IRQ_PROBE select GENERIC_IRQ_PROBE
select GENERIC_IRQ_RESERVATION_MODE
select GENERIC_IRQ_SHOW select GENERIC_IRQ_SHOW
select GENERIC_PENDING_IRQ if SMP select GENERIC_PENDING_IRQ if SMP
select GENERIC_SMP_IDLE_THREAD select GENERIC_SMP_IDLE_THREAD
......
This diff is collapsed.
...@@ -393,7 +393,7 @@ static inline void set_desc_limit(struct desc_struct *desc, unsigned long limit) ...@@ -393,7 +393,7 @@ static inline void set_desc_limit(struct desc_struct *desc, unsigned long limit)
void update_intr_gate(unsigned int n, const void *addr); void update_intr_gate(unsigned int n, const void *addr);
void alloc_intr_gate(unsigned int n, const void *addr); void alloc_intr_gate(unsigned int n, const void *addr);
extern unsigned long used_vectors[]; extern unsigned long system_vectors[];
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
DECLARE_PER_CPU(u32, debug_idt_ctr); DECLARE_PER_CPU(u32, debug_idt_ctr);
......
...@@ -16,6 +16,8 @@ ...@@ -16,6 +16,8 @@
#include <asm/irq_vectors.h> #include <asm/irq_vectors.h>
#define IRQ_MATRIX_BITS NR_VECTORS
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
#include <linux/percpu.h> #include <linux/percpu.h>
...@@ -123,15 +125,13 @@ struct irq_alloc_info { ...@@ -123,15 +125,13 @@ struct irq_alloc_info {
struct irq_cfg { struct irq_cfg {
unsigned int dest_apicid; unsigned int dest_apicid;
u8 vector; unsigned int vector;
u8 old_vector;
}; };
extern struct irq_cfg *irq_cfg(unsigned int irq); extern struct irq_cfg *irq_cfg(unsigned int irq);
extern struct irq_cfg *irqd_cfg(struct irq_data *irq_data); extern struct irq_cfg *irqd_cfg(struct irq_data *irq_data);
extern void lock_vector_lock(void); extern void lock_vector_lock(void);
extern void unlock_vector_lock(void); extern void unlock_vector_lock(void);
extern void setup_vector_irq(int cpu);
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
extern void send_cleanup_vector(struct irq_cfg *); extern void send_cleanup_vector(struct irq_cfg *);
extern void irq_complete_move(struct irq_cfg *cfg); extern void irq_complete_move(struct irq_cfg *cfg);
......
...@@ -193,7 +193,6 @@ static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg) ...@@ -193,7 +193,6 @@ static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
extern void setup_IO_APIC(void); extern void setup_IO_APIC(void);
extern void enable_IO_APIC(void); extern void enable_IO_APIC(void);
extern void disable_IO_APIC(void); extern void disable_IO_APIC(void);
extern void setup_ioapic_dest(void);
extern int IO_APIC_get_PCI_irq_vector(int bus, int devfn, int pin); extern int IO_APIC_get_PCI_irq_vector(int bus, int devfn, int pin);
extern void print_IO_APICs(void); extern void print_IO_APICs(void);
#else /* !CONFIG_X86_IO_APIC */ #else /* !CONFIG_X86_IO_APIC */
...@@ -233,7 +232,6 @@ static inline void io_apic_init_mappings(void) { } ...@@ -233,7 +232,6 @@ static inline void io_apic_init_mappings(void) { }
static inline void setup_IO_APIC(void) { } static inline void setup_IO_APIC(void) { }
static inline void enable_IO_APIC(void) { } static inline void enable_IO_APIC(void) { }
static inline void setup_ioapic_dest(void) { }
#endif #endif
......
...@@ -26,11 +26,7 @@ extern void irq_ctx_init(int cpu); ...@@ -26,11 +26,7 @@ extern void irq_ctx_init(int cpu);
struct irq_desc; struct irq_desc;
#ifdef CONFIG_HOTPLUG_CPU
#include <linux/cpumask.h>
extern int check_irq_vectors_for_cpu_disable(void);
extern void fixup_irqs(void); extern void fixup_irqs(void);
#endif
#ifdef CONFIG_HAVE_KVM #ifdef CONFIG_HAVE_KVM
extern void kvm_set_posted_intr_wakeup_handler(void (*handler)(void)); extern void kvm_set_posted_intr_wakeup_handler(void (*handler)(void));
......
...@@ -102,12 +102,8 @@ ...@@ -102,12 +102,8 @@
#define POSTED_INTR_NESTED_VECTOR 0xf0 #define POSTED_INTR_NESTED_VECTOR 0xf0
#endif #endif
/* #define MANAGED_IRQ_SHUTDOWN_VECTOR 0xef
* Local APIC timer IRQ vector is on a different priority level, #define LOCAL_TIMER_VECTOR 0xee
* to work around the 'lost local interrupt if more than 2 IRQ
* sources per level' errata.
*/
#define LOCAL_TIMER_VECTOR 0xef
#define NR_VECTORS 256 #define NR_VECTORS 256
......
...@@ -9,6 +9,7 @@ ...@@ -9,6 +9,7 @@
enum { enum {
/* Allocate contiguous CPU vectors */ /* Allocate contiguous CPU vectors */
X86_IRQ_ALLOC_CONTIGUOUS_VECTORS = 0x1, X86_IRQ_ALLOC_CONTIGUOUS_VECTORS = 0x1,
X86_IRQ_ALLOC_LEGACY = 0x2,
}; };
extern struct irq_domain *x86_vector_domain; extern struct irq_domain *x86_vector_domain;
......
...@@ -1419,7 +1419,7 @@ static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {} ...@@ -1419,7 +1419,7 @@ static inline void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) {}
static inline int kvm_cpu_get_apicid(int mps_cpu) static inline int kvm_cpu_get_apicid(int mps_cpu)
{ {
#ifdef CONFIG_X86_LOCAL_APIC #ifdef CONFIG_X86_LOCAL_APIC
return __default_cpu_present_to_apicid(mps_cpu); return default_cpu_present_to_apicid(mps_cpu);
#else #else
WARN_ON_ONCE(1); WARN_ON_ONCE(1);
return BAD_APICID; return BAD_APICID;
......
...@@ -138,6 +138,254 @@ DEFINE_IRQ_VECTOR_EVENT(deferred_error_apic); ...@@ -138,6 +138,254 @@ DEFINE_IRQ_VECTOR_EVENT(deferred_error_apic);
DEFINE_IRQ_VECTOR_EVENT(thermal_apic); DEFINE_IRQ_VECTOR_EVENT(thermal_apic);
#endif #endif
TRACE_EVENT(vector_config,
TP_PROTO(unsigned int irq, unsigned int vector,
unsigned int cpu, unsigned int apicdest),
TP_ARGS(irq, vector, cpu, apicdest),
TP_STRUCT__entry(
__field( unsigned int, irq )
__field( unsigned int, vector )
__field( unsigned int, cpu )
__field( unsigned int, apicdest )
),
TP_fast_assign(
__entry->irq = irq;
__entry->vector = vector;
__entry->cpu = cpu;
__entry->apicdest = apicdest;
),
TP_printk("irq=%u vector=%u cpu=%u apicdest=0x%08x",
__entry->irq, __entry->vector, __entry->cpu,
__entry->apicdest)
);
DECLARE_EVENT_CLASS(vector_mod,
TP_PROTO(unsigned int irq, unsigned int vector,
unsigned int cpu, unsigned int prev_vector,
unsigned int prev_cpu),
TP_ARGS(irq, vector, cpu, prev_vector, prev_cpu),
TP_STRUCT__entry(
__field( unsigned int, irq )
__field( unsigned int, vector )
__field( unsigned int, cpu )
__field( unsigned int, prev_vector )
__field( unsigned int, prev_cpu )
),
TP_fast_assign(
__entry->irq = irq;
__entry->vector = vector;
__entry->cpu = cpu;
__entry->prev_vector = prev_vector;
__entry->prev_cpu = prev_cpu;
),
TP_printk("irq=%u vector=%u cpu=%u prev_vector=%u prev_cpu=%u",
__entry->irq, __entry->vector, __entry->cpu,
__entry->prev_vector, __entry->prev_cpu)
);
#define DEFINE_IRQ_VECTOR_MOD_EVENT(name) \
DEFINE_EVENT_FN(vector_mod, name, \
TP_PROTO(unsigned int irq, unsigned int vector, \
unsigned int cpu, unsigned int prev_vector, \
unsigned int prev_cpu), \
TP_ARGS(irq, vector, cpu, prev_vector, prev_cpu), NULL, NULL); \
DEFINE_IRQ_VECTOR_MOD_EVENT(vector_update);
DEFINE_IRQ_VECTOR_MOD_EVENT(vector_clear);
DECLARE_EVENT_CLASS(vector_reserve,
TP_PROTO(unsigned int irq, int ret),
TP_ARGS(irq, ret),
TP_STRUCT__entry(
__field( unsigned int, irq )
__field( int, ret )
),
TP_fast_assign(
__entry->irq = irq;
__entry->ret = ret;
),
TP_printk("irq=%u ret=%d", __entry->irq, __entry->ret)
);
#define DEFINE_IRQ_VECTOR_RESERVE_EVENT(name) \
DEFINE_EVENT_FN(vector_reserve, name, \
TP_PROTO(unsigned int irq, int ret), \
TP_ARGS(irq, ret), NULL, NULL); \
DEFINE_IRQ_VECTOR_RESERVE_EVENT(vector_reserve_managed);
DEFINE_IRQ_VECTOR_RESERVE_EVENT(vector_reserve);
TRACE_EVENT(vector_alloc,
TP_PROTO(unsigned int irq, unsigned int vector, bool reserved,
int ret),
TP_ARGS(irq, vector, ret, reserved),
TP_STRUCT__entry(
__field( unsigned int, irq )
__field( unsigned int, vector )
__field( bool, reserved )
__field( int, ret )
),
TP_fast_assign(
__entry->irq = irq;
__entry->vector = ret < 0 ? 0 : vector;
__entry->reserved = reserved;
__entry->ret = ret > 0 ? 0 : ret;
),
TP_printk("irq=%u vector=%u reserved=%d ret=%d",
__entry->irq, __entry->vector,
__entry->reserved, __entry->ret)
);
TRACE_EVENT(vector_alloc_managed,
TP_PROTO(unsigned int irq, unsigned int vector,
int ret),
TP_ARGS(irq, vector, ret),
TP_STRUCT__entry(
__field( unsigned int, irq )
__field( unsigned int, vector )
__field( int, ret )
),
TP_fast_assign(
__entry->irq = irq;
__entry->vector = ret < 0 ? 0 : vector;
__entry->ret = ret > 0 ? 0 : ret;
),
TP_printk("irq=%u vector=%u ret=%d",
__entry->irq, __entry->vector, __entry->ret)
);
DECLARE_EVENT_CLASS(vector_activate,
TP_PROTO(unsigned int irq, bool is_managed, bool can_reserve,
bool early),
TP_ARGS(irq, is_managed, can_reserve, early),
TP_STRUCT__entry(
__field( unsigned int, irq )
__field( bool, is_managed )
__field( bool, can_reserve )
__field( bool, early )
),
TP_fast_assign(
__entry->irq = irq;
__entry->is_managed = is_managed;
__entry->can_reserve = can_reserve;
__entry->early = early;
),
TP_printk("irq=%u is_managed=%d can_reserve=%d early=%d",
__entry->irq, __entry->is_managed, __entry->can_reserve,
__entry->early)
);
#define DEFINE_IRQ_VECTOR_ACTIVATE_EVENT(name) \
DEFINE_EVENT_FN(vector_activate, name, \
TP_PROTO(unsigned int irq, bool is_managed, \
bool can_reserve, bool early), \
TP_ARGS(irq, is_managed, can_reserve, early), NULL, NULL); \
DEFINE_IRQ_VECTOR_ACTIVATE_EVENT(vector_activate);
DEFINE_IRQ_VECTOR_ACTIVATE_EVENT(vector_deactivate);
TRACE_EVENT(vector_teardown,
TP_PROTO(unsigned int irq, bool is_managed, bool has_reserved),
TP_ARGS(irq, is_managed, has_reserved),
TP_STRUCT__entry(
__field( unsigned int, irq )
__field( bool, is_managed )
__field( bool, has_reserved )
),
TP_fast_assign(
__entry->irq = irq;
__entry->is_managed = is_managed;
__entry->has_reserved = has_reserved;
),
TP_printk("irq=%u is_managed=%d has_reserved=%d",
__entry->irq, __entry->is_managed, __entry->has_reserved)
);
TRACE_EVENT(vector_setup,
TP_PROTO(unsigned int irq, bool is_legacy, int ret),
TP_ARGS(irq, is_legacy, ret),
TP_STRUCT__entry(
__field( unsigned int, irq )
__field( bool, is_legacy )
__field( int, ret )
),
TP_fast_assign(
__entry->irq = irq;
__entry->is_legacy = is_legacy;
__entry->ret = ret;
),
TP_printk("irq=%u is_legacy=%d ret=%d",
__entry->irq, __entry->is_legacy, __entry->ret)
);
TRACE_EVENT(vector_free_moved,
TP_PROTO(unsigned int irq, unsigned int cpu, unsigned int vector,
bool is_managed),
TP_ARGS(irq, cpu, vector, is_managed),
TP_STRUCT__entry(
__field( unsigned int, irq )
__field( unsigned int, cpu )
__field( unsigned int, vector )
__field( bool, is_managed )
),
TP_fast_assign(
__entry->irq = irq;
__entry->cpu = cpu;
__entry->vector = vector;
__entry->is_managed = is_managed;
),
TP_printk("irq=%u cpu=%u vector=%u is_managed=%d",
__entry->irq, __entry->cpu, __entry->vector,
__entry->is_managed)
);
#endif /* CONFIG_X86_LOCAL_APIC */ #endif /* CONFIG_X86_LOCAL_APIC */
#undef TRACE_INCLUDE_PATH #undef TRACE_INCLUDE_PATH
......
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Common bits for X2APIC cluster/physical modes.
*/
#ifndef _ASM_X86_X2APIC_H
#define _ASM_X86_X2APIC_H
#include <asm/apic.h>
#include <asm/ipi.h>
#include <linux/cpumask.h>
static int x2apic_apic_id_valid(int apicid)
{
return 1;
}
static int x2apic_apic_id_registered(void)
{
return 1;
}
static void
__x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest)
{
unsigned long cfg = __prepare_ICR(0, vector, dest);
native_x2apic_icr_write(cfg, apicid);
}
static unsigned int x2apic_get_apic_id(unsigned long id)
{
return id;
}
static unsigned long x2apic_set_apic_id(unsigned int id)
{
return id;
}
static int x2apic_phys_pkg_id(int initial_apicid, int index_msb)
{
return initial_apicid >> index_msb;
}
static void x2apic_send_IPI_self(int vector)
{
apic_write(APIC_SELF_IPI, vector);
}
#endif /* _ASM_X86_X2APIC_H */
...@@ -51,11 +51,13 @@ struct x86_init_resources { ...@@ -51,11 +51,13 @@ struct x86_init_resources {
* are set up. * are set up.
* @intr_init: interrupt init code * @intr_init: interrupt init code
* @trap_init: platform specific trap setup * @trap_init: platform specific trap setup
* @intr_mode_init: interrupt delivery mode setup
*/ */
struct x86_init_irqs { struct x86_init_irqs {
void (*pre_vector_init)(void); void (*pre_vector_init)(void);
void (*intr_init)(void); void (*intr_init)(void);
void (*trap_init)(void); void (*trap_init)(void);
void (*intr_mode_init)(void);
}; };
/** /**
......
...@@ -7,7 +7,7 @@ ...@@ -7,7 +7,7 @@
# In particualr, smp_apic_timer_interrupt() is called in random places. # In particualr, smp_apic_timer_interrupt() is called in random places.
KCOV_INSTRUMENT := n KCOV_INSTRUMENT := n
obj-$(CONFIG_X86_LOCAL_APIC) += apic.o apic_noop.o ipi.o vector.o obj-$(CONFIG_X86_LOCAL_APIC) += apic.o apic_common.o apic_noop.o ipi.o vector.o
obj-y += hw_nmi.o obj-y += hw_nmi.o
obj-$(CONFIG_X86_IO_APIC) += io_apic.o obj-$(CONFIG_X86_IO_APIC) += io_apic.o
......
...@@ -211,11 +211,7 @@ static inline int lapic_get_version(void) ...@@ -211,11 +211,7 @@ static inline int lapic_get_version(void)
*/ */
static inline int lapic_is_integrated(void) static inline int lapic_is_integrated(void)
{ {
#ifdef CONFIG_X86_64
return 1;
#else
return APIC_INTEGRATED(lapic_get_version()); return APIC_INTEGRATED(lapic_get_version());
#endif
} }
/* /*
...@@ -298,14 +294,11 @@ int get_physical_broadcast(void) ...@@ -298,14 +294,11 @@ int get_physical_broadcast(void)
*/ */
int lapic_get_maxlvt(void) int lapic_get_maxlvt(void)
{ {
unsigned int v;
v = apic_read(APIC_LVR);
/* /*
* - we always have APIC integrated on 64bit mode * - we always have APIC integrated on 64bit mode
* - 82489DXs do not report # of LVT entries * - 82489DXs do not report # of LVT entries
*/ */
return APIC_INTEGRATED(GET_APIC_VERSION(v)) ? GET_APIC_MAXLVT(v) : 2; return lapic_is_integrated() ? GET_APIC_MAXLVT(apic_read(APIC_LVR)) : 2;
} }
/* /*
...@@ -1229,53 +1222,100 @@ void __init sync_Arb_IDs(void) ...@@ -1229,53 +1222,100 @@ void __init sync_Arb_IDs(void)
APIC_INT_LEVELTRIG | APIC_DM_INIT); APIC_INT_LEVELTRIG | APIC_DM_INIT);
} }
/* enum apic_intr_mode_id apic_intr_mode;
* An initial setup of the virtual wire mode.
*/ static int __init apic_intr_mode_select(void)
void __init init_bsp_APIC(void)
{ {
unsigned int value; /* Check kernel option */
if (disable_apic) {
pr_info("APIC disabled via kernel command line\n");
return APIC_PIC;
}
/* /* Check BIOS */
* Don't do the setup now if we have a SMP BIOS as the #ifdef CONFIG_X86_64
* through-I/O-APIC virtual wire mode might be active. /* On 64-bit, the APIC must be integrated, Check local APIC only */
*/ if (!boot_cpu_has(X86_FEATURE_APIC)) {
if (smp_found_config || !boot_cpu_has(X86_FEATURE_APIC)) disable_apic = 1;
return; pr_info("APIC disabled by BIOS\n");
return APIC_PIC;
}
#else
/* On 32-bit, the APIC may be integrated APIC or 82489DX */
/* /* Neither 82489DX nor integrated APIC ? */
* Do not trust the local APIC being empty at bootup. if (!boot_cpu_has(X86_FEATURE_APIC) && !smp_found_config) {
*/ disable_apic = 1;
clear_local_APIC(); return APIC_PIC;
}
/* /* If the BIOS pretends there is an integrated APIC ? */
* Enable APIC. if (!boot_cpu_has(X86_FEATURE_APIC) &&
*/ APIC_INTEGRATED(boot_cpu_apic_version)) {
value = apic_read(APIC_SPIV); disable_apic = 1;
value &= ~APIC_VECTOR_MASK; pr_err(FW_BUG "Local APIC %d not detected, force emulation\n",
value |= APIC_SPIV_APIC_ENABLED; boot_cpu_physical_apicid);
return APIC_PIC;
}
#endif
#ifdef CONFIG_X86_32 /* Check MP table or ACPI MADT configuration */
/* This bit is reserved on P4/Xeon and should be cleared */ if (!smp_found_config) {
if ((boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) && disable_ioapic_support();
(boot_cpu_data.x86 == 15)) if (!acpi_lapic) {
value &= ~APIC_SPIV_FOCUS_DISABLED; pr_info("APIC: ACPI MADT or MP tables are not detected\n");
else return APIC_VIRTUAL_WIRE_NO_CONFIG;
}
return APIC_VIRTUAL_WIRE;
}
#ifdef CONFIG_SMP
/* If SMP should be disabled, then really disable it! */
if (!setup_max_cpus) {
pr_info("APIC: SMP mode deactivated\n");
return APIC_SYMMETRIC_IO_NO_ROUTING;
}
if (read_apic_id() != boot_cpu_physical_apicid) {
panic("Boot APIC ID in local APIC unexpected (%d vs %d)",
read_apic_id(), boot_cpu_physical_apicid);
/* Or can we switch back to PIC here? */
}
#endif #endif
value |= APIC_SPIV_FOCUS_DISABLED;
value |= SPURIOUS_APIC_VECTOR;
apic_write(APIC_SPIV, value);
/* return APIC_SYMMETRIC_IO;
* Set up the virtual wire mode. }
*/
apic_write(APIC_LVT0, APIC_DM_EXTINT); /* Init the interrupt delivery mode for the BSP */
value = APIC_DM_NMI; void __init apic_intr_mode_init(void)
if (!lapic_is_integrated()) /* 82489DX */ {
value |= APIC_LVT_LEVEL_TRIGGER; bool upmode = IS_ENABLED(CONFIG_UP_LATE_INIT);
if (apic_extnmi == APIC_EXTNMI_NONE)
value |= APIC_LVT_MASKED; apic_intr_mode = apic_intr_mode_select();
apic_write(APIC_LVT1, value);
switch (apic_intr_mode) {
case APIC_PIC:
pr_info("APIC: Keep in PIC mode(8259)\n");
return;
case APIC_VIRTUAL_WIRE:
pr_info("APIC: Switch to virtual wire mode setup\n");
default_setup_apic_routing();
break;
case APIC_VIRTUAL_WIRE_NO_CONFIG:
pr_info("APIC: Switch to virtual wire mode setup with no configuration\n");
upmode = true;
default_setup_apic_routing();
break;
case APIC_SYMMETRIC_IO:
pr_info("APIC: Switch to symmetric I/O mode setup\n");
default_setup_apic_routing();
break;
case APIC_SYMMETRIC_IO_NO_ROUTING:
pr_info("APIC: Switch to symmetric I/O mode setup in no SMP routine\n");
break;
}
apic_bsp_setup(upmode);
} }
static void lapic_setup_esr(void) static void lapic_setup_esr(void)
...@@ -1499,7 +1539,9 @@ void setup_local_APIC(void) ...@@ -1499,7 +1539,9 @@ void setup_local_APIC(void)
value = APIC_DM_NMI; value = APIC_DM_NMI;
else else
value = APIC_DM_NMI | APIC_LVT_MASKED; value = APIC_DM_NMI | APIC_LVT_MASKED;
if (!lapic_is_integrated()) /* 82489DX */
/* Is 82489DX ? */
if (!lapic_is_integrated())
value |= APIC_LVT_LEVEL_TRIGGER; value |= APIC_LVT_LEVEL_TRIGGER;
apic_write(APIC_LVT1, value); apic_write(APIC_LVT1, value);
...@@ -1885,8 +1927,8 @@ void __init init_apic_mappings(void) ...@@ -1885,8 +1927,8 @@ void __init init_apic_mappings(void)
* yeah -- we lie about apic_version * yeah -- we lie about apic_version
* in case if apic was disabled via boot option * in case if apic was disabled via boot option
* but it's not a problem for SMP compiled kernel * but it's not a problem for SMP compiled kernel
* since smp_sanity_check is prepared for such a case * since apic_intr_mode_select is prepared for such
* and disable smp mode * a case and disable smp mode
*/ */
boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR)); boot_cpu_apic_version = GET_APIC_VERSION(apic_read(APIC_LVR));
} }
...@@ -2242,44 +2284,6 @@ int hard_smp_processor_id(void) ...@@ -2242,44 +2284,6 @@ int hard_smp_processor_id(void)
return read_apic_id(); return read_apic_id();
} }
void default_init_apic_ldr(void)
{
unsigned long val;
apic_write(APIC_DFR, APIC_DFR_VALUE);
val = apic_read(APIC_LDR) & ~APIC_LDR_MASK;
val |= SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
apic_write(APIC_LDR, val);
}
int default_cpu_mask_to_apicid(const struct cpumask *mask,
struct irq_data *irqdata,
unsigned int *apicid)
{
unsigned int cpu = cpumask_first(mask);
if (cpu >= nr_cpu_ids)
return -EINVAL;
*apicid = per_cpu(x86_cpu_to_apicid, cpu);
irq_data_update_effective_affinity(irqdata, cpumask_of(cpu));
return 0;
}
int flat_cpu_mask_to_apicid(const struct cpumask *mask,
struct irq_data *irqdata,
unsigned int *apicid)
{
struct cpumask *effmsk = irq_data_get_effective_affinity_mask(irqdata);
unsigned long cpu_mask = cpumask_bits(mask)[0] & APIC_ALL_CPUS;
if (!cpu_mask)
return -EINVAL;
*apicid = (unsigned int)cpu_mask;
cpumask_bits(effmsk)[0] = cpu_mask;
return 0;
}
/* /*
* Override the generic EOI implementation with an optimized version. * Override the generic EOI implementation with an optimized version.
* Only called during early boot when only one CPU is active and with * Only called during early boot when only one CPU is active and with
...@@ -2322,72 +2326,27 @@ static void __init apic_bsp_up_setup(void) ...@@ -2322,72 +2326,27 @@ static void __init apic_bsp_up_setup(void)
* Returns: * Returns:
* apic_id of BSP APIC * apic_id of BSP APIC
*/ */
int __init apic_bsp_setup(bool upmode) void __init apic_bsp_setup(bool upmode)
{ {
int id;
connect_bsp_APIC(); connect_bsp_APIC();
if (upmode) if (upmode)
apic_bsp_up_setup(); apic_bsp_up_setup();
setup_local_APIC(); setup_local_APIC();
if (x2apic_mode)
id = apic_read(APIC_LDR);
else
id = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));
enable_IO_APIC(); enable_IO_APIC();
end_local_APIC_setup(); end_local_APIC_setup();
irq_remap_enable_fault_handling(); irq_remap_enable_fault_handling();
setup_IO_APIC(); setup_IO_APIC();
/* Setup local timer */
x86_init.timers.setup_percpu_clockev();
return id;
}
/*
* This initializes the IO-APIC and APIC hardware if this is
* a UP kernel.
*/
int __init APIC_init_uniprocessor(void)
{
if (disable_apic) {
pr_info("Apic disabled\n");
return -1;
}
#ifdef CONFIG_X86_64
if (!boot_cpu_has(X86_FEATURE_APIC)) {
disable_apic = 1;
pr_info("Apic disabled by BIOS\n");
return -1;
}
#else
if (!smp_found_config && !boot_cpu_has(X86_FEATURE_APIC))
return -1;
/*
* Complain if the BIOS pretends there is one.
*/
if (!boot_cpu_has(X86_FEATURE_APIC) &&
APIC_INTEGRATED(boot_cpu_apic_version)) {
pr_err("BIOS bug, local APIC 0x%x not detected!...\n",
boot_cpu_physical_apicid);
return -1;
}
#endif
if (!smp_found_config)
disable_ioapic_support();
default_setup_apic_routing();
apic_bsp_setup(true);
return 0;
} }
#ifdef CONFIG_UP_LATE_INIT #ifdef CONFIG_UP_LATE_INIT
void __init up_late_init(void) void __init up_late_init(void)
{ {
APIC_init_uniprocessor(); if (apic_intr_mode == APIC_PIC)
return;
/* Setup local timer */
x86_init.timers.setup_percpu_clockev();
} }
#endif #endif
......
/*
* Common functions shared between the various APIC flavours
*
* SPDX-License-Identifier: GPL-2.0
*/
#include <linux/irq.h>
#include <asm/apic.h>
u32 apic_default_calc_apicid(unsigned int cpu)
{
return per_cpu(x86_cpu_to_apicid, cpu);
}
u32 apic_flat_calc_apicid(unsigned int cpu)
{
return 1U << cpu;
}
bool default_check_apicid_used(physid_mask_t *map, int apicid)
{
return physid_isset(apicid, *map);
}
void default_ioapic_phys_id_map(physid_mask_t *phys_map, physid_mask_t *retmap)
{
*retmap = *phys_map;
}
int default_cpu_present_to_apicid(int mps_cpu)
{
if (mps_cpu < nr_cpu_ids && cpu_present(mps_cpu))
return (int)per_cpu(x86_bios_cpu_apicid, mps_cpu);
else
return BAD_APICID;
}
EXPORT_SYMBOL_GPL(default_cpu_present_to_apicid);
int default_check_phys_apicid_present(int phys_apicid)
{
return physid_isset(phys_apicid, phys_cpu_present_map);
}
int default_apic_id_valid(int apicid)
{
return (apicid < 255);
}
...@@ -119,7 +119,7 @@ static unsigned int flat_get_apic_id(unsigned long x) ...@@ -119,7 +119,7 @@ static unsigned int flat_get_apic_id(unsigned long x)
return (x >> 24) & 0xFF; return (x >> 24) & 0xFF;
} }
static unsigned long set_apic_id(unsigned int id) static u32 set_apic_id(unsigned int id)
{ {
return (id & 0xFF) << 24; return (id & 0xFF) << 24;
} }
...@@ -154,12 +154,10 @@ static struct apic apic_flat __ro_after_init = { ...@@ -154,12 +154,10 @@ static struct apic apic_flat __ro_after_init = {
.irq_delivery_mode = dest_LowestPrio, .irq_delivery_mode = dest_LowestPrio,
.irq_dest_mode = 1, /* logical */ .irq_dest_mode = 1, /* logical */
.target_cpus = online_target_cpus,
.disable_esr = 0, .disable_esr = 0,
.dest_logical = APIC_DEST_LOGICAL, .dest_logical = APIC_DEST_LOGICAL,
.check_apicid_used = NULL, .check_apicid_used = NULL,
.vector_allocation_domain = flat_vector_allocation_domain,
.init_apic_ldr = flat_init_apic_ldr, .init_apic_ldr = flat_init_apic_ldr,
.ioapic_phys_id_map = NULL, .ioapic_phys_id_map = NULL,
...@@ -172,7 +170,7 @@ static struct apic apic_flat __ro_after_init = { ...@@ -172,7 +170,7 @@ static struct apic apic_flat __ro_after_init = {
.get_apic_id = flat_get_apic_id, .get_apic_id = flat_get_apic_id,
.set_apic_id = set_apic_id, .set_apic_id = set_apic_id,
.cpu_mask_to_apicid = flat_cpu_mask_to_apicid, .calc_dest_apicid = apic_flat_calc_apicid,
.send_IPI = default_send_IPI_single, .send_IPI = default_send_IPI_single,
.send_IPI_mask = flat_send_IPI_mask, .send_IPI_mask = flat_send_IPI_mask,
...@@ -249,12 +247,10 @@ static struct apic apic_physflat __ro_after_init = { ...@@ -249,12 +247,10 @@ static struct apic apic_physflat __ro_after_init = {
.irq_delivery_mode = dest_Fixed, .irq_delivery_mode = dest_Fixed,
.irq_dest_mode = 0, /* physical */ .irq_dest_mode = 0, /* physical */
.target_cpus = online_target_cpus,
.disable_esr = 0, .disable_esr = 0,
.dest_logical = 0, .dest_logical = 0,
.check_apicid_used = NULL, .check_apicid_used = NULL,
.vector_allocation_domain = default_vector_allocation_domain,
/* not needed, but shouldn't hurt: */ /* not needed, but shouldn't hurt: */
.init_apic_ldr = flat_init_apic_ldr, .init_apic_ldr = flat_init_apic_ldr,
...@@ -268,7 +264,7 @@ static struct apic apic_physflat __ro_after_init = { ...@@ -268,7 +264,7 @@ static struct apic apic_physflat __ro_after_init = {
.get_apic_id = flat_get_apic_id, .get_apic_id = flat_get_apic_id,
.set_apic_id = set_apic_id, .set_apic_id = set_apic_id,
.cpu_mask_to_apicid = default_cpu_mask_to_apicid, .calc_dest_apicid = apic_default_calc_apicid,
.send_IPI = default_send_IPI_single_phys, .send_IPI = default_send_IPI_single_phys,
.send_IPI_mask = default_send_IPI_mask_sequence_phys, .send_IPI_mask = default_send_IPI_mask_sequence_phys,
......
...@@ -84,20 +84,6 @@ static int noop_apic_id_registered(void) ...@@ -84,20 +84,6 @@ static int noop_apic_id_registered(void)
return physid_isset(0, phys_cpu_present_map); return physid_isset(0, phys_cpu_present_map);
} }
static const struct cpumask *noop_target_cpus(void)
{
/* only BSP here */
return cpumask_of(0);
}
static void noop_vector_allocation_domain(int cpu, struct cpumask *retmask,
const struct cpumask *mask)
{
if (cpu != 0)
pr_warning("APIC: Vector allocated for non-BSP cpu\n");
cpumask_copy(retmask, cpumask_of(cpu));
}
static u32 noop_apic_read(u32 reg) static u32 noop_apic_read(u32 reg)
{ {
WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !disable_apic); WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !disable_apic);
...@@ -109,6 +95,13 @@ static void noop_apic_write(u32 reg, u32 v) ...@@ -109,6 +95,13 @@ static void noop_apic_write(u32 reg, u32 v)
WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !disable_apic); WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_APIC) && !disable_apic);
} }
#ifdef CONFIG_X86_32
static int noop_x86_32_early_logical_apicid(int cpu)
{
return BAD_APICID;
}
#endif
struct apic apic_noop __ro_after_init = { struct apic apic_noop __ro_after_init = {
.name = "noop", .name = "noop",
.probe = noop_probe, .probe = noop_probe,
...@@ -121,12 +114,10 @@ struct apic apic_noop __ro_after_init = { ...@@ -121,12 +114,10 @@ struct apic apic_noop __ro_after_init = {
/* logical delivery broadcast to all CPUs: */ /* logical delivery broadcast to all CPUs: */
.irq_dest_mode = 1, .irq_dest_mode = 1,
.target_cpus = noop_target_cpus,
.disable_esr = 0, .disable_esr = 0,
.dest_logical = APIC_DEST_LOGICAL, .dest_logical = APIC_DEST_LOGICAL,
.check_apicid_used = default_check_apicid_used, .check_apicid_used = default_check_apicid_used,
.vector_allocation_domain = noop_vector_allocation_domain,
.init_apic_ldr = noop_init_apic_ldr, .init_apic_ldr = noop_init_apic_ldr,
.ioapic_phys_id_map = default_ioapic_phys_id_map, .ioapic_phys_id_map = default_ioapic_phys_id_map,
...@@ -142,7 +133,7 @@ struct apic apic_noop __ro_after_init = { ...@@ -142,7 +133,7 @@ struct apic apic_noop __ro_after_init = {
.get_apic_id = noop_get_apic_id, .get_apic_id = noop_get_apic_id,
.set_apic_id = NULL, .set_apic_id = NULL,
.cpu_mask_to_apicid = flat_cpu_mask_to_apicid, .calc_dest_apicid = apic_flat_calc_apicid,
.send_IPI = noop_send_IPI, .send_IPI = noop_send_IPI,
.send_IPI_mask = noop_send_IPI_mask, .send_IPI_mask = noop_send_IPI_mask,
......
...@@ -38,7 +38,7 @@ static unsigned int numachip1_get_apic_id(unsigned long x) ...@@ -38,7 +38,7 @@ static unsigned int numachip1_get_apic_id(unsigned long x)
return id; return id;
} }
static unsigned long numachip1_set_apic_id(unsigned int id) static u32 numachip1_set_apic_id(unsigned int id)
{ {
return (id & 0xff) << 24; return (id & 0xff) << 24;
} }
...@@ -51,7 +51,7 @@ static unsigned int numachip2_get_apic_id(unsigned long x) ...@@ -51,7 +51,7 @@ static unsigned int numachip2_get_apic_id(unsigned long x)
return ((mcfg >> (28 - 8)) & 0xfff00) | (x >> 24); return ((mcfg >> (28 - 8)) & 0xfff00) | (x >> 24);
} }
static unsigned long numachip2_set_apic_id(unsigned int id) static u32 numachip2_set_apic_id(unsigned int id)
{ {
return id << 24; return id << 24;
} }
...@@ -249,12 +249,10 @@ static const struct apic apic_numachip1 __refconst = { ...@@ -249,12 +249,10 @@ static const struct apic apic_numachip1 __refconst = {
.irq_delivery_mode = dest_Fixed, .irq_delivery_mode = dest_Fixed,
.irq_dest_mode = 0, /* physical */ .irq_dest_mode = 0, /* physical */
.target_cpus = online_target_cpus,
.disable_esr = 0, .disable_esr = 0,
.dest_logical = 0, .dest_logical = 0,
.check_apicid_used = NULL, .check_apicid_used = NULL,
.vector_allocation_domain = default_vector_allocation_domain,
.init_apic_ldr = flat_init_apic_ldr, .init_apic_ldr = flat_init_apic_ldr,
.ioapic_phys_id_map = NULL, .ioapic_phys_id_map = NULL,
...@@ -267,7 +265,7 @@ static const struct apic apic_numachip1 __refconst = { ...@@ -267,7 +265,7 @@ static const struct apic apic_numachip1 __refconst = {
.get_apic_id = numachip1_get_apic_id, .get_apic_id = numachip1_get_apic_id,
.set_apic_id = numachip1_set_apic_id, .set_apic_id = numachip1_set_apic_id,
.cpu_mask_to_apicid = default_cpu_mask_to_apicid, .calc_dest_apicid = apic_default_calc_apicid,
.send_IPI = numachip_send_IPI_one, .send_IPI = numachip_send_IPI_one,
.send_IPI_mask = numachip_send_IPI_mask, .send_IPI_mask = numachip_send_IPI_mask,
...@@ -300,12 +298,10 @@ static const struct apic apic_numachip2 __refconst = { ...@@ -300,12 +298,10 @@ static const struct apic apic_numachip2 __refconst = {
.irq_delivery_mode = dest_Fixed, .irq_delivery_mode = dest_Fixed,
.irq_dest_mode = 0, /* physical */ .irq_dest_mode = 0, /* physical */
.target_cpus = online_target_cpus,
.disable_esr = 0, .disable_esr = 0,
.dest_logical = 0, .dest_logical = 0,
.check_apicid_used = NULL, .check_apicid_used = NULL,
.vector_allocation_domain = default_vector_allocation_domain,
.init_apic_ldr = flat_init_apic_ldr, .init_apic_ldr = flat_init_apic_ldr,
.ioapic_phys_id_map = NULL, .ioapic_phys_id_map = NULL,
...@@ -318,7 +314,7 @@ static const struct apic apic_numachip2 __refconst = { ...@@ -318,7 +314,7 @@ static const struct apic apic_numachip2 __refconst = {
.get_apic_id = numachip2_get_apic_id, .get_apic_id = numachip2_get_apic_id,
.set_apic_id = numachip2_set_apic_id, .set_apic_id = numachip2_set_apic_id,
.cpu_mask_to_apicid = default_cpu_mask_to_apicid, .calc_dest_apicid = apic_default_calc_apicid,
.send_IPI = numachip_send_IPI_one, .send_IPI = numachip_send_IPI_one,
.send_IPI_mask = numachip_send_IPI_mask, .send_IPI_mask = numachip_send_IPI_mask,
......
...@@ -27,9 +27,9 @@ static int bigsmp_apic_id_registered(void) ...@@ -27,9 +27,9 @@ static int bigsmp_apic_id_registered(void)
return 1; return 1;
} }
static unsigned long bigsmp_check_apicid_used(physid_mask_t *map, int apicid) static bool bigsmp_check_apicid_used(physid_mask_t *map, int apicid)
{ {
return 0; return false;
} }
static int bigsmp_early_logical_apicid(int cpu) static int bigsmp_early_logical_apicid(int cpu)
...@@ -155,12 +155,10 @@ static struct apic apic_bigsmp __ro_after_init = { ...@@ -155,12 +155,10 @@ static struct apic apic_bigsmp __ro_after_init = {
/* phys delivery to target CPU: */ /* phys delivery to target CPU: */
.irq_dest_mode = 0, .irq_dest_mode = 0,
.target_cpus = default_target_cpus,
.disable_esr = 1, .disable_esr = 1,
.dest_logical = 0, .dest_logical = 0,
.check_apicid_used = bigsmp_check_apicid_used, .check_apicid_used = bigsmp_check_apicid_used,
.vector_allocation_domain = default_vector_allocation_domain,
.init_apic_ldr = bigsmp_init_apic_ldr, .init_apic_ldr = bigsmp_init_apic_ldr,
.ioapic_phys_id_map = bigsmp_ioapic_phys_id_map, .ioapic_phys_id_map = bigsmp_ioapic_phys_id_map,
...@@ -173,7 +171,7 @@ static struct apic apic_bigsmp __ro_after_init = { ...@@ -173,7 +171,7 @@ static struct apic apic_bigsmp __ro_after_init = {
.get_apic_id = bigsmp_get_apic_id, .get_apic_id = bigsmp_get_apic_id,
.set_apic_id = NULL, .set_apic_id = NULL,
.cpu_mask_to_apicid = default_cpu_mask_to_apicid, .calc_dest_apicid = apic_default_calc_apicid,
.send_IPI = default_send_IPI_single_phys, .send_IPI = default_send_IPI_single_phys,
.send_IPI_mask = default_send_IPI_mask_sequence_phys, .send_IPI_mask = default_send_IPI_mask_sequence_phys,
......
...@@ -1014,6 +1014,7 @@ static int alloc_isa_irq_from_domain(struct irq_domain *domain, ...@@ -1014,6 +1014,7 @@ static int alloc_isa_irq_from_domain(struct irq_domain *domain,
info->ioapic_pin)) info->ioapic_pin))
return -ENOMEM; return -ENOMEM;
} else { } else {
info->flags |= X86_IRQ_ALLOC_LEGACY;
irq = __irq_domain_alloc_irqs(domain, irq, 1, node, info, true, irq = __irq_domain_alloc_irqs(domain, irq, 1, node, info, true,
NULL); NULL);
if (irq >= 0) { if (irq >= 0) {
...@@ -1586,6 +1587,43 @@ static int __init notimercheck(char *s) ...@@ -1586,6 +1587,43 @@ static int __init notimercheck(char *s)
} }
__setup("no_timer_check", notimercheck); __setup("no_timer_check", notimercheck);
static void __init delay_with_tsc(void)
{
unsigned long long start, now;
unsigned long end = jiffies + 4;
start = rdtsc();
/*
* We don't know the TSC frequency yet, but waiting for
* 40000000000/HZ TSC cycles is safe:
* 4 GHz == 10 jiffies
* 1 GHz == 40 jiffies
*/
do {
rep_nop();
now = rdtsc();
} while ((now - start) < 40000000000UL / HZ &&
time_before_eq(jiffies, end));
}
static void __init delay_without_tsc(void)
{
unsigned long end = jiffies + 4;
int band = 1;
/*
* We don't know any frequency yet, but waiting for
* 40940000000/HZ cycles is safe:
* 4 GHz == 10 jiffies
* 1 GHz == 40 jiffies
* 1 << 1 + 1 << 2 +...+ 1 << 11 = 4094
*/
do {
__delay(((1U << band++) * 10000000UL) / HZ);
} while (band < 12 && time_before_eq(jiffies, end));
}
/* /*
* There is a nasty bug in some older SMP boards, their mptable lies * There is a nasty bug in some older SMP boards, their mptable lies
* about the timer IRQ. We do the following to work around the situation: * about the timer IRQ. We do the following to work around the situation:
...@@ -1604,8 +1642,12 @@ static int __init timer_irq_works(void) ...@@ -1604,8 +1642,12 @@ static int __init timer_irq_works(void)
local_save_flags(flags); local_save_flags(flags);
local_irq_enable(); local_irq_enable();
/* Let ten ticks pass... */
mdelay((10 * 1000) / HZ); if (boot_cpu_has(X86_FEATURE_TSC))
delay_with_tsc();
else
delay_without_tsc();
local_irq_restore(flags); local_irq_restore(flags);
/* /*
...@@ -1821,26 +1863,36 @@ static void ioapic_ir_ack_level(struct irq_data *irq_data) ...@@ -1821,26 +1863,36 @@ static void ioapic_ir_ack_level(struct irq_data *irq_data)
eoi_ioapic_pin(data->entry.vector, data); eoi_ioapic_pin(data->entry.vector, data);
} }
static void ioapic_configure_entry(struct irq_data *irqd)
{
struct mp_chip_data *mpd = irqd->chip_data;
struct irq_cfg *cfg = irqd_cfg(irqd);
struct irq_pin_list *entry;
/*
* Only update when the parent is the vector domain, don't touch it
* if the parent is the remapping domain. Check the installed
* ioapic chip to verify that.
*/
if (irqd->chip == &ioapic_chip) {
mpd->entry.dest = cfg->dest_apicid;
mpd->entry.vector = cfg->vector;
}
for_each_irq_pin(entry, mpd->irq_2_pin)
__ioapic_write_entry(entry->apic, entry->pin, mpd->entry);
}
static int ioapic_set_affinity(struct irq_data *irq_data, static int ioapic_set_affinity(struct irq_data *irq_data,
const struct cpumask *mask, bool force) const struct cpumask *mask, bool force)
{ {
struct irq_data *parent = irq_data->parent_data; struct irq_data *parent = irq_data->parent_data;
struct mp_chip_data *data = irq_data->chip_data;
struct irq_pin_list *entry;
struct irq_cfg *cfg;
unsigned long flags; unsigned long flags;
int ret; int ret;
ret = parent->chip->irq_set_affinity(parent, mask, force); ret = parent->chip->irq_set_affinity(parent, mask, force);
raw_spin_lock_irqsave(&ioapic_lock, flags); raw_spin_lock_irqsave(&ioapic_lock, flags);
if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE) { if (ret >= 0 && ret != IRQ_SET_MASK_OK_DONE)
cfg = irqd_cfg(irq_data); ioapic_configure_entry(irq_data);
data->entry.dest = cfg->dest_apicid;
data->entry.vector = cfg->vector;
for_each_irq_pin(entry, data->irq_2_pin)
__ioapic_write_entry(entry->apic, entry->pin,
data->entry);
}
raw_spin_unlock_irqrestore(&ioapic_lock, flags); raw_spin_unlock_irqrestore(&ioapic_lock, flags);
return ret; return ret;
...@@ -2513,52 +2565,9 @@ int acpi_get_override_irq(u32 gsi, int *trigger, int *polarity) ...@@ -2513,52 +2565,9 @@ int acpi_get_override_irq(u32 gsi, int *trigger, int *polarity)
} }
/* /*
* This function currently is only a helper for the i386 smp boot process where * This function updates target affinity of IOAPIC interrupts to include
* we need to reprogram the ioredtbls to cater for the cpus which have come online * the CPUs which came online during SMP bringup.
* so mask in all cases should simply be apic->target_cpus()
*/ */
#ifdef CONFIG_SMP
void __init setup_ioapic_dest(void)
{
int pin, ioapic, irq, irq_entry;
const struct cpumask *mask;
struct irq_desc *desc;
struct irq_data *idata;
struct irq_chip *chip;
if (skip_ioapic_setup == 1)
return;
for_each_ioapic_pin(ioapic, pin) {
irq_entry = find_irq_entry(ioapic, pin, mp_INT);
if (irq_entry == -1)
continue;
irq = pin_2_irq(irq_entry, ioapic, pin, 0);
if (irq < 0 || !mp_init_irq_at_boot(ioapic, irq))
continue;
desc = irq_to_desc(irq);
raw_spin_lock_irq(&desc->lock);
idata = irq_desc_get_irq_data(desc);
/*
* Honour affinities which have been set in early boot
*/
if (!irqd_can_balance(idata) || irqd_affinity_was_set(idata))
mask = irq_data_get_affinity_mask(idata);
else
mask = apic->target_cpus();
chip = irq_data_get_irq_chip(idata);
/* Might be lapic_chip for irq 0 */
if (chip->irq_set_affinity)
chip->irq_set_affinity(idata, mask, false);
raw_spin_unlock_irq(&desc->lock);
}
}
#endif
#define IOAPIC_RESOURCE_NAME_SIZE 11 #define IOAPIC_RESOURCE_NAME_SIZE 11
static struct resource *ioapic_resources; static struct resource *ioapic_resources;
...@@ -2982,12 +2991,9 @@ int mp_irqdomain_activate(struct irq_domain *domain, ...@@ -2982,12 +2991,9 @@ int mp_irqdomain_activate(struct irq_domain *domain,
struct irq_data *irq_data, bool early) struct irq_data *irq_data, bool early)
{ {
unsigned long flags; unsigned long flags;
struct irq_pin_list *entry;
struct mp_chip_data *data = irq_data->chip_data;
raw_spin_lock_irqsave(&ioapic_lock, flags); raw_spin_lock_irqsave(&ioapic_lock, flags);
for_each_irq_pin(entry, data->irq_2_pin) ioapic_configure_entry(irq_data);
__ioapic_write_entry(entry->apic, entry->pin, data->entry);
raw_spin_unlock_irqrestore(&ioapic_lock, flags); raw_spin_unlock_irqrestore(&ioapic_lock, flags);
return 0; return 0;
} }
......
...@@ -66,6 +66,31 @@ static void setup_apic_flat_routing(void) ...@@ -66,6 +66,31 @@ static void setup_apic_flat_routing(void)
#endif #endif
} }
static int default_apic_id_registered(void)
{
return physid_isset(read_apic_id(), phys_cpu_present_map);
}
/*
* Set up the logical destination ID. Intel recommends to set DFR, LDR and
* TPR before enabling an APIC. See e.g. "AP-388 82489DX User's Manual"
* (Intel document number 292116).
*/
static void default_init_apic_ldr(void)
{
unsigned long val;
apic_write(APIC_DFR, APIC_DFR_VALUE);
val = apic_read(APIC_LDR) & ~APIC_LDR_MASK;
val |= SET_APIC_LOGICAL_ID(1UL << smp_processor_id());
apic_write(APIC_LDR, val);
}
static int default_phys_pkg_id(int cpuid_apic, int index_msb)
{
return cpuid_apic >> index_msb;
}
/* should be called last. */ /* should be called last. */
static int probe_default(void) static int probe_default(void)
{ {
...@@ -84,12 +109,10 @@ static struct apic apic_default __ro_after_init = { ...@@ -84,12 +109,10 @@ static struct apic apic_default __ro_after_init = {
/* logical delivery broadcast to all CPUs: */ /* logical delivery broadcast to all CPUs: */
.irq_dest_mode = 1, .irq_dest_mode = 1,
.target_cpus = default_target_cpus,
.disable_esr = 0, .disable_esr = 0,
.dest_logical = APIC_DEST_LOGICAL, .dest_logical = APIC_DEST_LOGICAL,
.check_apicid_used = default_check_apicid_used, .check_apicid_used = default_check_apicid_used,
.vector_allocation_domain = flat_vector_allocation_domain,
.init_apic_ldr = default_init_apic_ldr, .init_apic_ldr = default_init_apic_ldr,
.ioapic_phys_id_map = default_ioapic_phys_id_map, .ioapic_phys_id_map = default_ioapic_phys_id_map,
...@@ -102,7 +125,7 @@ static struct apic apic_default __ro_after_init = { ...@@ -102,7 +125,7 @@ static struct apic apic_default __ro_after_init = {
.get_apic_id = default_get_apic_id, .get_apic_id = default_get_apic_id,
.set_apic_id = NULL, .set_apic_id = NULL,
.cpu_mask_to_apicid = flat_cpu_mask_to_apicid, .calc_dest_apicid = apic_flat_calc_apicid,
.send_IPI = default_send_IPI_single, .send_IPI = default_send_IPI_single,
.send_IPI_mask = default_send_IPI_mask_logical, .send_IPI_mask = default_send_IPI_mask_logical,
......
This diff is collapsed.
/* Common bits for X2APIC cluster/physical modes. */
int x2apic_apic_id_valid(int apicid);
int x2apic_apic_id_registered(void);
void __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest);
unsigned int x2apic_get_apic_id(unsigned long id);
u32 x2apic_set_apic_id(unsigned int id);
int x2apic_phys_pkg_id(int initial_apicid, int index_msb);
void x2apic_send_IPI_self(int vector);
...@@ -9,22 +9,24 @@ ...@@ -9,22 +9,24 @@
#include <linux/cpu.h> #include <linux/cpu.h>
#include <asm/smp.h> #include <asm/smp.h>
#include <asm/x2apic.h> #include "x2apic.h"
struct cluster_mask {
unsigned int clusterid;
int node;
struct cpumask mask;
};
static DEFINE_PER_CPU(u32, x86_cpu_to_logical_apicid); static DEFINE_PER_CPU(u32, x86_cpu_to_logical_apicid);
static DEFINE_PER_CPU(cpumask_var_t, cpus_in_cluster);
static DEFINE_PER_CPU(cpumask_var_t, ipi_mask); static DEFINE_PER_CPU(cpumask_var_t, ipi_mask);
static DEFINE_PER_CPU(struct cluster_mask *, cluster_masks);
static struct cluster_mask *cluster_hotplug_mask;
static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id) static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
{ {
return x2apic_enabled(); return x2apic_enabled();
} }
static inline u32 x2apic_cluster(int cpu)
{
return per_cpu(x86_cpu_to_logical_apicid, cpu) >> 16;
}
static void x2apic_send_IPI(int cpu, int vector) static void x2apic_send_IPI(int cpu, int vector)
{ {
u32 dest = per_cpu(x86_cpu_to_logical_apicid, cpu); u32 dest = per_cpu(x86_cpu_to_logical_apicid, cpu);
...@@ -36,49 +38,34 @@ static void x2apic_send_IPI(int cpu, int vector) ...@@ -36,49 +38,34 @@ static void x2apic_send_IPI(int cpu, int vector)
static void static void
__x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest) __x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
{ {
struct cpumask *cpus_in_cluster_ptr; unsigned int cpu, clustercpu;
struct cpumask *ipi_mask_ptr; struct cpumask *tmpmsk;
unsigned int cpu, this_cpu;
unsigned long flags; unsigned long flags;
u32 dest; u32 dest;
x2apic_wrmsr_fence(); x2apic_wrmsr_fence();
local_irq_save(flags); local_irq_save(flags);
this_cpu = smp_processor_id(); tmpmsk = this_cpu_cpumask_var_ptr(ipi_mask);
cpumask_copy(tmpmsk, mask);
/* If IPI should not be sent to self, clear current CPU */
if (apic_dest != APIC_DEST_ALLINC)
cpumask_clear_cpu(smp_processor_id(), tmpmsk);
/* /* Collapse cpus in a cluster so a single IPI per cluster is sent */
* We are to modify mask, so we need an own copy for_each_cpu(cpu, tmpmsk) {
* and be sure it's manipulated with irq off. struct cluster_mask *cmsk = per_cpu(cluster_masks, cpu);
*/
ipi_mask_ptr = this_cpu_cpumask_var_ptr(ipi_mask);
cpumask_copy(ipi_mask_ptr, mask);
/*
* The idea is to send one IPI per cluster.
*/
for_each_cpu(cpu, ipi_mask_ptr) {
unsigned long i;
cpus_in_cluster_ptr = per_cpu(cpus_in_cluster, cpu);
dest = 0; dest = 0;
for_each_cpu_and(clustercpu, tmpmsk, &cmsk->mask)
/* Collect cpus in cluster. */ dest |= per_cpu(x86_cpu_to_logical_apicid, clustercpu);
for_each_cpu_and(i, ipi_mask_ptr, cpus_in_cluster_ptr) {
if (apic_dest == APIC_DEST_ALLINC || i != this_cpu)
dest |= per_cpu(x86_cpu_to_logical_apicid, i);
}
if (!dest) if (!dest)
continue; continue;
__x2apic_send_IPI_dest(dest, vector, apic->dest_logical); __x2apic_send_IPI_dest(dest, vector, apic->dest_logical);
/* /* Remove cluster CPUs from tmpmask */
* Cluster sibling cpus should be discared now so cpumask_andnot(tmpmsk, tmpmsk, &cmsk->mask);
* we would not send IPI them second time.
*/
cpumask_andnot(ipi_mask_ptr, ipi_mask_ptr, cpus_in_cluster_ptr);
} }
local_irq_restore(flags); local_irq_restore(flags);
...@@ -105,125 +92,90 @@ static void x2apic_send_IPI_all(int vector) ...@@ -105,125 +92,90 @@ static void x2apic_send_IPI_all(int vector)
__x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLINC); __x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLINC);
} }
static int static u32 x2apic_calc_apicid(unsigned int cpu)
x2apic_cpu_mask_to_apicid(const struct cpumask *mask, struct irq_data *irqdata,
unsigned int *apicid)
{ {
struct cpumask *effmsk = irq_data_get_effective_affinity_mask(irqdata); return per_cpu(x86_cpu_to_logical_apicid, cpu);
unsigned int cpu;
u32 dest = 0;
u16 cluster;
cpu = cpumask_first(mask);
if (cpu >= nr_cpu_ids)
return -EINVAL;
dest = per_cpu(x86_cpu_to_logical_apicid, cpu);
cluster = x2apic_cluster(cpu);
cpumask_clear(effmsk);
for_each_cpu(cpu, mask) {
if (cluster != x2apic_cluster(cpu))
continue;
dest |= per_cpu(x86_cpu_to_logical_apicid, cpu);
cpumask_set_cpu(cpu, effmsk);
}
*apicid = dest;
return 0;
} }
static void init_x2apic_ldr(void) static void init_x2apic_ldr(void)
{ {
unsigned int this_cpu = smp_processor_id(); struct cluster_mask *cmsk = this_cpu_read(cluster_masks);
u32 cluster, apicid = apic_read(APIC_LDR);
unsigned int cpu; unsigned int cpu;
per_cpu(x86_cpu_to_logical_apicid, this_cpu) = apic_read(APIC_LDR); this_cpu_write(x86_cpu_to_logical_apicid, apicid);
if (cmsk)
goto update;
cpumask_set_cpu(this_cpu, per_cpu(cpus_in_cluster, this_cpu)); cluster = apicid >> 16;
for_each_online_cpu(cpu) { for_each_online_cpu(cpu) {
if (x2apic_cluster(this_cpu) != x2apic_cluster(cpu)) cmsk = per_cpu(cluster_masks, cpu);
continue; /* Matching cluster found. Link and update it. */
cpumask_set_cpu(this_cpu, per_cpu(cpus_in_cluster, cpu)); if (cmsk && cmsk->clusterid == cluster)
cpumask_set_cpu(cpu, per_cpu(cpus_in_cluster, this_cpu)); goto update;
} }
cmsk = cluster_hotplug_mask;
cluster_hotplug_mask = NULL;
update:
this_cpu_write(cluster_masks, cmsk);
cpumask_set_cpu(smp_processor_id(), &cmsk->mask);
} }
/* static int alloc_clustermask(unsigned int cpu, int node)
* At CPU state changes, update the x2apic cluster sibling info.
*/
static int x2apic_prepare_cpu(unsigned int cpu)
{ {
if (!zalloc_cpumask_var(&per_cpu(cpus_in_cluster, cpu), GFP_KERNEL)) if (per_cpu(cluster_masks, cpu))
return -ENOMEM; return 0;
/*
* If a hotplug spare mask exists, check whether it's on the right
* node. If not, free it and allocate a new one.
*/
if (cluster_hotplug_mask) {
if (cluster_hotplug_mask->node == node)
return 0;
kfree(cluster_hotplug_mask);
}
if (!zalloc_cpumask_var(&per_cpu(ipi_mask, cpu), GFP_KERNEL)) { cluster_hotplug_mask = kzalloc_node(sizeof(*cluster_hotplug_mask),
free_cpumask_var(per_cpu(cpus_in_cluster, cpu)); GFP_KERNEL, node);
if (!cluster_hotplug_mask)
return -ENOMEM; return -ENOMEM;
} cluster_hotplug_mask->node = node;
return 0;
}
static int x2apic_prepare_cpu(unsigned int cpu)
{
if (alloc_clustermask(cpu, cpu_to_node(cpu)) < 0)
return -ENOMEM;
if (!zalloc_cpumask_var(&per_cpu(ipi_mask, cpu), GFP_KERNEL))
return -ENOMEM;
return 0; return 0;
} }
static int x2apic_dead_cpu(unsigned int this_cpu) static int x2apic_dead_cpu(unsigned int dead_cpu)
{ {
int cpu; struct cluster_mask *cmsk = per_cpu(cluster_masks, dead_cpu);
for_each_online_cpu(cpu) { cpumask_clear_cpu(dead_cpu, &cmsk->mask);
if (x2apic_cluster(this_cpu) != x2apic_cluster(cpu)) free_cpumask_var(per_cpu(ipi_mask, dead_cpu));
continue;
cpumask_clear_cpu(this_cpu, per_cpu(cpus_in_cluster, cpu));
cpumask_clear_cpu(cpu, per_cpu(cpus_in_cluster, this_cpu));
}
free_cpumask_var(per_cpu(cpus_in_cluster, this_cpu));
free_cpumask_var(per_cpu(ipi_mask, this_cpu));
return 0; return 0;
} }
static int x2apic_cluster_probe(void) static int x2apic_cluster_probe(void)
{ {
int cpu = smp_processor_id();
int ret;
if (!x2apic_mode) if (!x2apic_mode)
return 0; return 0;
ret = cpuhp_setup_state(CPUHP_X2APIC_PREPARE, "x86/x2apic:prepare", if (cpuhp_setup_state(CPUHP_X2APIC_PREPARE, "x86/x2apic:prepare",
x2apic_prepare_cpu, x2apic_dead_cpu); x2apic_prepare_cpu, x2apic_dead_cpu) < 0) {
if (ret < 0) {
pr_err("Failed to register X2APIC_PREPARE\n"); pr_err("Failed to register X2APIC_PREPARE\n");
return 0; return 0;
} }
cpumask_set_cpu(cpu, per_cpu(cpus_in_cluster, cpu)); init_x2apic_ldr();
return 1; return 1;
} }
static const struct cpumask *x2apic_cluster_target_cpus(void)
{
return cpu_all_mask;
}
/*
* Each x2apic cluster is an allocation domain.
*/
static void cluster_vector_allocation_domain(int cpu, struct cpumask *retmask,
const struct cpumask *mask)
{
/*
* To minimize vector pressure, default case of boot, device bringup
* etc will use a single cpu for the interrupt destination.
*
* On explicit migration requests coming from irqbalance etc,
* interrupts will be routed to the x2apic cluster (cluster-id
* derived from the first cpu in the mask) members specified
* in the mask.
*/
if (mask == x2apic_cluster_target_cpus())
cpumask_copy(retmask, cpumask_of(cpu));
else
cpumask_and(retmask, mask, per_cpu(cpus_in_cluster, cpu));
}
static struct apic apic_x2apic_cluster __ro_after_init = { static struct apic apic_x2apic_cluster __ro_after_init = {
.name = "cluster x2apic", .name = "cluster x2apic",
...@@ -235,12 +187,10 @@ static struct apic apic_x2apic_cluster __ro_after_init = { ...@@ -235,12 +187,10 @@ static struct apic apic_x2apic_cluster __ro_after_init = {
.irq_delivery_mode = dest_LowestPrio, .irq_delivery_mode = dest_LowestPrio,
.irq_dest_mode = 1, /* logical */ .irq_dest_mode = 1, /* logical */
.target_cpus = x2apic_cluster_target_cpus,
.disable_esr = 0, .disable_esr = 0,
.dest_logical = APIC_DEST_LOGICAL, .dest_logical = APIC_DEST_LOGICAL,
.check_apicid_used = NULL, .check_apicid_used = NULL,
.vector_allocation_domain = cluster_vector_allocation_domain,
.init_apic_ldr = init_x2apic_ldr, .init_apic_ldr = init_x2apic_ldr,
.ioapic_phys_id_map = NULL, .ioapic_phys_id_map = NULL,
...@@ -253,7 +203,7 @@ static struct apic apic_x2apic_cluster __ro_after_init = { ...@@ -253,7 +203,7 @@ static struct apic apic_x2apic_cluster __ro_after_init = {
.get_apic_id = x2apic_get_apic_id, .get_apic_id = x2apic_get_apic_id,
.set_apic_id = x2apic_set_apic_id, .set_apic_id = x2apic_set_apic_id,
.cpu_mask_to_apicid = x2apic_cpu_mask_to_apicid, .calc_dest_apicid = x2apic_calc_apicid,
.send_IPI = x2apic_send_IPI, .send_IPI = x2apic_send_IPI,
.send_IPI_mask = x2apic_send_IPI_mask, .send_IPI_mask = x2apic_send_IPI_mask,
......
...@@ -7,7 +7,8 @@ ...@@ -7,7 +7,8 @@
#include <linux/dmar.h> #include <linux/dmar.h>
#include <asm/smp.h> #include <asm/smp.h>
#include <asm/x2apic.h> #include <asm/ipi.h>
#include "x2apic.h"
int x2apic_phys; int x2apic_phys;
...@@ -99,6 +100,43 @@ static int x2apic_phys_probe(void) ...@@ -99,6 +100,43 @@ static int x2apic_phys_probe(void)
return apic == &apic_x2apic_phys; return apic == &apic_x2apic_phys;
} }
/* Common x2apic functions, also used by x2apic_cluster */
int x2apic_apic_id_valid(int apicid)
{
return 1;
}
int x2apic_apic_id_registered(void)
{
return 1;
}
void __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest)
{
unsigned long cfg = __prepare_ICR(0, vector, dest);
native_x2apic_icr_write(cfg, apicid);
}
unsigned int x2apic_get_apic_id(unsigned long id)
{
return id;
}
u32 x2apic_set_apic_id(unsigned int id)
{
return id;
}
int x2apic_phys_pkg_id(int initial_apicid, int index_msb)
{
return initial_apicid >> index_msb;
}
void x2apic_send_IPI_self(int vector)
{
apic_write(APIC_SELF_IPI, vector);
}
static struct apic apic_x2apic_phys __ro_after_init = { static struct apic apic_x2apic_phys __ro_after_init = {
.name = "physical x2apic", .name = "physical x2apic",
...@@ -110,12 +148,10 @@ static struct apic apic_x2apic_phys __ro_after_init = { ...@@ -110,12 +148,10 @@ static struct apic apic_x2apic_phys __ro_after_init = {
.irq_delivery_mode = dest_Fixed, .irq_delivery_mode = dest_Fixed,
.irq_dest_mode = 0, /* physical */ .irq_dest_mode = 0, /* physical */
.target_cpus = online_target_cpus,
.disable_esr = 0, .disable_esr = 0,
.dest_logical = 0, .dest_logical = 0,
.check_apicid_used = NULL, .check_apicid_used = NULL,
.vector_allocation_domain = default_vector_allocation_domain,
.init_apic_ldr = init_x2apic_ldr, .init_apic_ldr = init_x2apic_ldr,
.ioapic_phys_id_map = NULL, .ioapic_phys_id_map = NULL,
...@@ -128,7 +164,7 @@ static struct apic apic_x2apic_phys __ro_after_init = { ...@@ -128,7 +164,7 @@ static struct apic apic_x2apic_phys __ro_after_init = {
.get_apic_id = x2apic_get_apic_id, .get_apic_id = x2apic_get_apic_id,
.set_apic_id = x2apic_set_apic_id, .set_apic_id = x2apic_set_apic_id,
.cpu_mask_to_apicid = default_cpu_mask_to_apicid, .calc_dest_apicid = apic_default_calc_apicid,
.send_IPI = x2apic_send_IPI, .send_IPI = x2apic_send_IPI,
.send_IPI_mask = x2apic_send_IPI_mask, .send_IPI_mask = x2apic_send_IPI_mask,
......
...@@ -525,16 +525,9 @@ static void uv_init_apic_ldr(void) ...@@ -525,16 +525,9 @@ static void uv_init_apic_ldr(void)
{ {
} }
static int static u32 apic_uv_calc_apicid(unsigned int cpu)
uv_cpu_mask_to_apicid(const struct cpumask *mask, struct irq_data *irqdata,
unsigned int *apicid)
{ {
int ret = default_cpu_mask_to_apicid(mask, irqdata, apicid); return apic_default_calc_apicid(cpu) | uv_apicid_hibits;
if (!ret)
*apicid |= uv_apicid_hibits;
return ret;
} }
static unsigned int x2apic_get_apic_id(unsigned long x) static unsigned int x2apic_get_apic_id(unsigned long x)
...@@ -547,7 +540,7 @@ static unsigned int x2apic_get_apic_id(unsigned long x) ...@@ -547,7 +540,7 @@ static unsigned int x2apic_get_apic_id(unsigned long x)
return id; return id;
} }
static unsigned long set_apic_id(unsigned int id) static u32 set_apic_id(unsigned int id)
{ {
/* CHECKME: Do we need to mask out the xapic extra bits? */ /* CHECKME: Do we need to mask out the xapic extra bits? */
return id; return id;
...@@ -584,12 +577,10 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { ...@@ -584,12 +577,10 @@ static struct apic apic_x2apic_uv_x __ro_after_init = {
.irq_delivery_mode = dest_Fixed, .irq_delivery_mode = dest_Fixed,
.irq_dest_mode = 0, /* Physical */ .irq_dest_mode = 0, /* Physical */
.target_cpus = online_target_cpus,
.disable_esr = 0, .disable_esr = 0,
.dest_logical = APIC_DEST_LOGICAL, .dest_logical = APIC_DEST_LOGICAL,
.check_apicid_used = NULL, .check_apicid_used = NULL,
.vector_allocation_domain = default_vector_allocation_domain,
.init_apic_ldr = uv_init_apic_ldr, .init_apic_ldr = uv_init_apic_ldr,
.ioapic_phys_id_map = NULL, .ioapic_phys_id_map = NULL,
...@@ -602,7 +593,7 @@ static struct apic apic_x2apic_uv_x __ro_after_init = { ...@@ -602,7 +593,7 @@ static struct apic apic_x2apic_uv_x __ro_after_init = {
.get_apic_id = x2apic_get_apic_id, .get_apic_id = x2apic_get_apic_id,
.set_apic_id = set_apic_id, .set_apic_id = set_apic_id,
.cpu_mask_to_apicid = uv_cpu_mask_to_apicid, .calc_dest_apicid = apic_uv_calc_apicid,
.send_IPI = uv_send_IPI_one, .send_IPI = uv_send_IPI_one,
.send_IPI_mask = uv_send_IPI_mask, .send_IPI_mask = uv_send_IPI_mask,
......
...@@ -114,6 +114,7 @@ static void make_8259A_irq(unsigned int irq) ...@@ -114,6 +114,7 @@ static void make_8259A_irq(unsigned int irq)
io_apic_irqs &= ~(1<<irq); io_apic_irqs &= ~(1<<irq);
irq_set_chip_and_handler(irq, &i8259A_chip, handle_level_irq); irq_set_chip_and_handler(irq, &i8259A_chip, handle_level_irq);
enable_irq(irq); enable_irq(irq);
lapic_assign_legacy_vector(irq, true);
} }
/* /*
......
...@@ -223,7 +223,7 @@ idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sy ...@@ -223,7 +223,7 @@ idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sy
idt_init_desc(&desc, t); idt_init_desc(&desc, t);
write_idt_entry(idt, t->vector, &desc); write_idt_entry(idt, t->vector, &desc);
if (sys) if (sys)
set_bit(t->vector, used_vectors); set_bit(t->vector, system_vectors);
} }
} }
...@@ -311,14 +311,14 @@ void __init idt_setup_apic_and_irq_gates(void) ...@@ -311,14 +311,14 @@ void __init idt_setup_apic_and_irq_gates(void)
idt_setup_from_table(idt_table, apic_idts, ARRAY_SIZE(apic_idts), true); idt_setup_from_table(idt_table, apic_idts, ARRAY_SIZE(apic_idts), true);
for_each_clear_bit_from(i, used_vectors, FIRST_SYSTEM_VECTOR) { for_each_clear_bit_from(i, system_vectors, FIRST_SYSTEM_VECTOR) {
entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR); entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR);
set_intr_gate(i, entry); set_intr_gate(i, entry);
} }
for_each_clear_bit_from(i, used_vectors, NR_VECTORS) { for_each_clear_bit_from(i, system_vectors, NR_VECTORS) {
#ifdef CONFIG_X86_LOCAL_APIC #ifdef CONFIG_X86_LOCAL_APIC
set_bit(i, used_vectors); set_bit(i, system_vectors);
set_intr_gate(i, spurious_interrupt); set_intr_gate(i, spurious_interrupt);
#else #else
entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR); entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR);
...@@ -356,7 +356,7 @@ void idt_invalidate(void *addr) ...@@ -356,7 +356,7 @@ void idt_invalidate(void *addr)
void __init update_intr_gate(unsigned int n, const void *addr) void __init update_intr_gate(unsigned int n, const void *addr)
{ {
if (WARN_ON_ONCE(!test_bit(n, used_vectors))) if (WARN_ON_ONCE(!test_bit(n, system_vectors)))
return; return;
set_intr_gate(n, addr); set_intr_gate(n, addr);
} }
...@@ -364,6 +364,6 @@ void __init update_intr_gate(unsigned int n, const void *addr) ...@@ -364,6 +364,6 @@ void __init update_intr_gate(unsigned int n, const void *addr)
void alloc_intr_gate(unsigned int n, const void *addr) void alloc_intr_gate(unsigned int n, const void *addr)
{ {
BUG_ON(n < FIRST_SYSTEM_VECTOR); BUG_ON(n < FIRST_SYSTEM_VECTOR);
if (!test_and_set_bit(n, used_vectors)) if (!test_and_set_bit(n, system_vectors))
set_intr_gate(n, addr); set_intr_gate(n, addr);
} }
...@@ -134,7 +134,7 @@ int arch_show_interrupts(struct seq_file *p, int prec) ...@@ -134,7 +134,7 @@ int arch_show_interrupts(struct seq_file *p, int prec)
seq_puts(p, " Machine check polls\n"); seq_puts(p, " Machine check polls\n");
#endif #endif
#if IS_ENABLED(CONFIG_HYPERV) || defined(CONFIG_XEN) #if IS_ENABLED(CONFIG_HYPERV) || defined(CONFIG_XEN)
if (test_bit(HYPERVISOR_CALLBACK_VECTOR, used_vectors)) { if (test_bit(HYPERVISOR_CALLBACK_VECTOR, system_vectors)) {
seq_printf(p, "%*s: ", prec, "HYP"); seq_printf(p, "%*s: ", prec, "HYP");
for_each_online_cpu(j) for_each_online_cpu(j)
seq_printf(p, "%10u ", seq_printf(p, "%10u ",
...@@ -333,105 +333,6 @@ __visible void smp_kvm_posted_intr_nested_ipi(struct pt_regs *regs) ...@@ -333,105 +333,6 @@ __visible void smp_kvm_posted_intr_nested_ipi(struct pt_regs *regs)
#ifdef CONFIG_HOTPLUG_CPU #ifdef CONFIG_HOTPLUG_CPU
/* These two declarations are only used in check_irq_vectors_for_cpu_disable()
* below, which is protected by stop_machine(). Putting them on the stack
* results in a stack frame overflow. Dynamically allocating could result in a
* failure so declare these two cpumasks as global.
*/
static struct cpumask affinity_new, online_new;
/*
* This cpu is going to be removed and its vectors migrated to the remaining
* online cpus. Check to see if there are enough vectors in the remaining cpus.
* This function is protected by stop_machine().
*/
int check_irq_vectors_for_cpu_disable(void)
{
unsigned int this_cpu, vector, this_count, count;
struct irq_desc *desc;
struct irq_data *data;
int cpu;
this_cpu = smp_processor_id();
cpumask_copy(&online_new, cpu_online_mask);
cpumask_clear_cpu(this_cpu, &online_new);
this_count = 0;
for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
desc = __this_cpu_read(vector_irq[vector]);
if (IS_ERR_OR_NULL(desc))
continue;
/*
* Protect against concurrent action removal, affinity
* changes etc.
*/
raw_spin_lock(&desc->lock);
data = irq_desc_get_irq_data(desc);
cpumask_copy(&affinity_new,
irq_data_get_affinity_mask(data));
cpumask_clear_cpu(this_cpu, &affinity_new);
/* Do not count inactive or per-cpu irqs. */
if (!irq_desc_has_action(desc) || irqd_is_per_cpu(data)) {
raw_spin_unlock(&desc->lock);
continue;
}
raw_spin_unlock(&desc->lock);
/*
* A single irq may be mapped to multiple cpu's
* vector_irq[] (for example IOAPIC cluster mode). In
* this case we have two possibilities:
*
* 1) the resulting affinity mask is empty; that is
* this the down'd cpu is the last cpu in the irq's
* affinity mask, or
*
* 2) the resulting affinity mask is no longer a
* subset of the online cpus but the affinity mask is
* not zero; that is the down'd cpu is the last online
* cpu in a user set affinity mask.
*/
if (cpumask_empty(&affinity_new) ||
!cpumask_subset(&affinity_new, &online_new))
this_count++;
}
/* No need to check any further. */
if (!this_count)
return 0;
count = 0;
for_each_online_cpu(cpu) {
if (cpu == this_cpu)
continue;
/*
* We scan from FIRST_EXTERNAL_VECTOR to first system
* vector. If the vector is marked in the used vectors
* bitmap or an irq is assigned to it, we don't count
* it as available.
*
* As this is an inaccurate snapshot anyway, we can do
* this w/o holding vector_lock.
*/
for (vector = FIRST_EXTERNAL_VECTOR;
vector < FIRST_SYSTEM_VECTOR; vector++) {
if (!test_bit(vector, used_vectors) &&
IS_ERR_OR_NULL(per_cpu(vector_irq, cpu)[vector])) {
if (++count == this_count)
return 0;
}
}
}
if (count < this_count) {
pr_warn("CPU %d disable failed: CPU has %u vectors assigned and there are only %u available.\n",
this_cpu, this_count, count);
return -ERANGE;
}
return 0;
}
/* A cpu has been removed from cpu_online_mask. Reset irq affinities. */ /* A cpu has been removed from cpu_online_mask. Reset irq affinities. */
void fixup_irqs(void) void fixup_irqs(void)
{ {
......
...@@ -61,9 +61,6 @@ void __init init_ISA_irqs(void) ...@@ -61,9 +61,6 @@ void __init init_ISA_irqs(void)
struct irq_chip *chip = legacy_pic->chip; struct irq_chip *chip = legacy_pic->chip;
int i; int i;
#if defined(CONFIG_X86_64) || defined(CONFIG_X86_LOCAL_APIC)
init_bsp_APIC();
#endif
legacy_pic->init(0); legacy_pic->init(0);
for (i = 0; i < nr_legacy_irqs(); i++) for (i = 0; i < nr_legacy_irqs(); i++)
...@@ -94,6 +91,7 @@ void __init native_init_IRQ(void) ...@@ -94,6 +91,7 @@ void __init native_init_IRQ(void)
x86_init.irqs.pre_vector_init(); x86_init.irqs.pre_vector_init();
idt_setup_apic_and_irq_gates(); idt_setup_apic_and_irq_gates();
lapic_assign_system_vectors();
if (!acpi_ioapic && !of_ioapic && nr_legacy_irqs()) if (!acpi_ioapic && !of_ioapic && nr_legacy_irqs())
setup_irq(2, &irq2); setup_irq(2, &irq2);
......
...@@ -136,18 +136,6 @@ RESERVE_BRK(dmi_alloc, 65536); ...@@ -136,18 +136,6 @@ RESERVE_BRK(dmi_alloc, 65536);
static __initdata unsigned long _brk_start = (unsigned long)__brk_base; static __initdata unsigned long _brk_start = (unsigned long)__brk_base;
unsigned long _brk_end = (unsigned long)__brk_base; unsigned long _brk_end = (unsigned long)__brk_base;
#ifdef CONFIG_X86_64
int default_cpu_present_to_apicid(int mps_cpu)
{
return __default_cpu_present_to_apicid(mps_cpu);
}
int default_check_phys_apicid_present(int phys_apicid)
{
return __default_check_phys_apicid_present(phys_apicid);
}
#endif
struct boot_params boot_params; struct boot_params boot_params;
/* /*
......
...@@ -256,14 +256,14 @@ static void notrace start_secondary(void *unused) ...@@ -256,14 +256,14 @@ static void notrace start_secondary(void *unused)
check_tsc_sync_target(); check_tsc_sync_target();
/* /*
* Lock vector_lock and initialize the vectors on this cpu * Lock vector_lock, set CPU online and bring the vector
* before setting the cpu online. We must set it online with * allocator online. Online must be set with vector_lock held
* vector_lock held to prevent a concurrent setup/teardown * to prevent a concurrent irq setup/teardown from seeing a
* from seeing a half valid vector space. * half valid vector space.
*/ */
lock_vector_lock(); lock_vector_lock();
setup_vector_irq(smp_processor_id());
set_cpu_online(smp_processor_id(), true); set_cpu_online(smp_processor_id(), true);
lapic_online();
unlock_vector_lock(); unlock_vector_lock();
cpu_set_state_online(smp_processor_id()); cpu_set_state_online(smp_processor_id());
x86_platform.nmi_init(); x86_platform.nmi_init();
...@@ -1191,17 +1191,10 @@ static __init void disable_smp(void) ...@@ -1191,17 +1191,10 @@ static __init void disable_smp(void)
cpumask_set_cpu(0, topology_core_cpumask(0)); cpumask_set_cpu(0, topology_core_cpumask(0));
} }
enum {
SMP_OK,
SMP_NO_CONFIG,
SMP_NO_APIC,
SMP_FORCE_UP,
};
/* /*
* Various sanity checks. * Various sanity checks.
*/ */
static int __init smp_sanity_check(unsigned max_cpus) static void __init smp_sanity_check(void)
{ {
preempt_disable(); preempt_disable();
...@@ -1238,16 +1231,6 @@ static int __init smp_sanity_check(unsigned max_cpus) ...@@ -1238,16 +1231,6 @@ static int __init smp_sanity_check(unsigned max_cpus)
physid_set(hard_smp_processor_id(), phys_cpu_present_map); physid_set(hard_smp_processor_id(), phys_cpu_present_map);
} }
/*
* If we couldn't find an SMP configuration at boot time,
* get out of here now!
*/
if (!smp_found_config && !acpi_lapic) {
preempt_enable();
pr_notice("SMP motherboard not detected\n");
return SMP_NO_CONFIG;
}
/* /*
* Should not be necessary because the MP table should list the boot * Should not be necessary because the MP table should list the boot
* CPU too, but we do it for the sake of robustness anyway. * CPU too, but we do it for the sake of robustness anyway.
...@@ -1258,29 +1241,6 @@ static int __init smp_sanity_check(unsigned max_cpus) ...@@ -1258,29 +1241,6 @@ static int __init smp_sanity_check(unsigned max_cpus)
physid_set(hard_smp_processor_id(), phys_cpu_present_map); physid_set(hard_smp_processor_id(), phys_cpu_present_map);
} }
preempt_enable(); preempt_enable();
/*
* If we couldn't find a local APIC, then get out of here now!
*/
if (APIC_INTEGRATED(boot_cpu_apic_version) &&
!boot_cpu_has(X86_FEATURE_APIC)) {
if (!disable_apic) {
pr_err("BIOS bug, local APIC #%d not detected!...\n",
boot_cpu_physical_apicid);
pr_err("... forcing use of dummy APIC emulation (tell your hw vendor)\n");
}
return SMP_NO_APIC;
}
/*
* If SMP should be disabled, then really disable it!
*/
if (!max_cpus) {
pr_info("SMP mode deactivated\n");
return SMP_FORCE_UP;
}
return SMP_OK;
} }
static void __init smp_cpu_index_default(void) static void __init smp_cpu_index_default(void)
...@@ -1295,9 +1255,18 @@ static void __init smp_cpu_index_default(void) ...@@ -1295,9 +1255,18 @@ static void __init smp_cpu_index_default(void)
} }
} }
static void __init smp_get_logical_apicid(void)
{
if (x2apic_mode)
cpu0_logical_apicid = apic_read(APIC_LDR);
else
cpu0_logical_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));
}
/* /*
* Prepare for SMP bootup. The MP table or ACPI has been read * Prepare for SMP bootup.
* earlier. Just do some sanity checking here and enable APIC mode. * @max_cpus: configured maximum number of CPUs, It is a legacy parameter
* for common interface support.
*/ */
void __init native_smp_prepare_cpus(unsigned int max_cpus) void __init native_smp_prepare_cpus(unsigned int max_cpus)
{ {
...@@ -1329,31 +1298,27 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus) ...@@ -1329,31 +1298,27 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus)
set_cpu_sibling_map(0); set_cpu_sibling_map(0);
switch (smp_sanity_check(max_cpus)) { smp_sanity_check();
case SMP_NO_CONFIG:
disable_smp(); switch (apic_intr_mode) {
if (APIC_init_uniprocessor()) case APIC_PIC:
pr_notice("Local APIC not detected. Using dummy APIC emulation.\n"); case APIC_VIRTUAL_WIRE_NO_CONFIG:
return;
case SMP_NO_APIC:
disable_smp(); disable_smp();
return; return;
case SMP_FORCE_UP: case APIC_SYMMETRIC_IO_NO_ROUTING:
disable_smp(); disable_smp();
apic_bsp_setup(false); /* Setup local timer */
x86_init.timers.setup_percpu_clockev();
return; return;
case SMP_OK: case APIC_VIRTUAL_WIRE:
case APIC_SYMMETRIC_IO:
break; break;
} }
if (read_apic_id() != boot_cpu_physical_apicid) { /* Setup local timer */
panic("Boot APIC ID in local APIC unexpected (%d vs %d)", x86_init.timers.setup_percpu_clockev();
read_apic_id(), boot_cpu_physical_apicid);
/* Or can we switch back to PIC here? */
}
default_setup_apic_routing(); smp_get_logical_apicid();
cpu0_logical_apicid = apic_bsp_setup(false);
pr_info("CPU0: "); pr_info("CPU0: ");
print_cpu_info(&cpu_data(0)); print_cpu_info(&cpu_data(0));
...@@ -1398,7 +1363,6 @@ void __init native_smp_cpus_done(unsigned int max_cpus) ...@@ -1398,7 +1363,6 @@ void __init native_smp_cpus_done(unsigned int max_cpus)
nmi_selftest(); nmi_selftest();
impress_friends(); impress_friends();
setup_ioapic_dest();
mtrr_aps_init(); mtrr_aps_init();
} }
...@@ -1557,13 +1521,14 @@ void cpu_disable_common(void) ...@@ -1557,13 +1521,14 @@ void cpu_disable_common(void)
remove_cpu_from_maps(cpu); remove_cpu_from_maps(cpu);
unlock_vector_lock(); unlock_vector_lock();
fixup_irqs(); fixup_irqs();
lapic_offline();
} }
int native_cpu_disable(void) int native_cpu_disable(void)
{ {
int ret; int ret;
ret = check_irq_vectors_for_cpu_disable(); ret = lapic_can_unplug_cpu();
if (ret) if (ret)
return ret; return ret;
......
...@@ -85,6 +85,11 @@ void __init hpet_time_init(void) ...@@ -85,6 +85,11 @@ void __init hpet_time_init(void)
static __init void x86_late_time_init(void) static __init void x86_late_time_init(void)
{ {
x86_init.timers.timer_init(); x86_init.timers.timer_init();
/*
* After PIT/HPET timers init, select and setup
* the final interrupt mode for delivering IRQs.
*/
x86_init.irqs.intr_mode_init();
tsc_init(); tsc_init();
} }
......
...@@ -72,7 +72,7 @@ ...@@ -72,7 +72,7 @@
#include <asm/proto.h> #include <asm/proto.h>
#endif #endif
DECLARE_BITMAP(used_vectors, NR_VECTORS); DECLARE_BITMAP(system_vectors, NR_VECTORS);
static inline void cond_local_irq_enable(struct pt_regs *regs) static inline void cond_local_irq_enable(struct pt_regs *regs)
{ {
......
...@@ -26,9 +26,6 @@ ...@@ -26,9 +26,6 @@
#define TOPOLOGY_REGISTER_OFFSET 0x10 #define TOPOLOGY_REGISTER_OFFSET 0x10
/* Flag below is initialized once during vSMP PCI initialization. */
static int irq_routing_comply = 1;
#if defined CONFIG_PCI && defined CONFIG_PARAVIRT #if defined CONFIG_PCI && defined CONFIG_PARAVIRT
/* /*
* Interrupt control on vSMPowered systems: * Interrupt control on vSMPowered systems:
...@@ -105,9 +102,6 @@ static void __init set_vsmp_pv_ops(void) ...@@ -105,9 +102,6 @@ static void __init set_vsmp_pv_ops(void)
if (cap & ctl & BIT(8)) { if (cap & ctl & BIT(8)) {
ctl &= ~BIT(8); ctl &= ~BIT(8);
/* Interrupt routing set to ignore */
irq_routing_comply = 0;
#ifdef CONFIG_PROC_FS #ifdef CONFIG_PROC_FS
/* Don't let users change irq affinity via procfs */ /* Don't let users change irq affinity via procfs */
no_irq_affinity = 1; no_irq_affinity = 1;
...@@ -211,23 +205,10 @@ static int apicid_phys_pkg_id(int initial_apic_id, int index_msb) ...@@ -211,23 +205,10 @@ static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
return hard_smp_processor_id() >> index_msb; return hard_smp_processor_id() >> index_msb;
} }
/*
* In vSMP, all cpus should be capable of handling interrupts, regardless of
* the APIC used.
*/
static void fill_vector_allocation_domain(int cpu, struct cpumask *retmask,
const struct cpumask *mask)
{
cpumask_setall(retmask);
}
static void vsmp_apic_post_init(void) static void vsmp_apic_post_init(void)
{ {
/* need to update phys_pkg_id */ /* need to update phys_pkg_id */
apic->phys_pkg_id = apicid_phys_pkg_id; apic->phys_pkg_id = apicid_phys_pkg_id;
if (!irq_routing_comply)
apic->vector_allocation_domain = fill_vector_allocation_domain;
} }
void __init vsmp_init(void) void __init vsmp_init(void)
......
...@@ -57,6 +57,7 @@ struct x86_init_ops x86_init __initdata = { ...@@ -57,6 +57,7 @@ struct x86_init_ops x86_init __initdata = {
.pre_vector_init = init_ISA_irqs, .pre_vector_init = init_ISA_irqs,
.intr_init = native_init_IRQ, .intr_init = native_init_IRQ,
.trap_init = x86_init_noop, .trap_init = x86_init_noop,
.intr_mode_init = apic_intr_mode_init
}, },
.oem = { .oem = {
......
...@@ -31,7 +31,7 @@ static unsigned int xen_io_apic_read(unsigned apic, unsigned reg) ...@@ -31,7 +31,7 @@ static unsigned int xen_io_apic_read(unsigned apic, unsigned reg)
return 0xfd; return 0xfd;
} }
static unsigned long xen_set_apic_id(unsigned int x) static u32 xen_set_apic_id(unsigned int x)
{ {
WARN_ON(1); WARN_ON(1);
return x; return x;
...@@ -161,12 +161,10 @@ static struct apic xen_pv_apic = { ...@@ -161,12 +161,10 @@ static struct apic xen_pv_apic = {
/* .irq_delivery_mode - used in native_compose_msi_msg only */ /* .irq_delivery_mode - used in native_compose_msi_msg only */
/* .irq_dest_mode - used in native_compose_msi_msg only */ /* .irq_dest_mode - used in native_compose_msi_msg only */
.target_cpus = default_target_cpus,
.disable_esr = 0, .disable_esr = 0,
/* .dest_logical - default_send_IPI_ use it but we use our own. */ /* .dest_logical - default_send_IPI_ use it but we use our own. */
.check_apicid_used = default_check_apicid_used, /* Used on 32-bit */ .check_apicid_used = default_check_apicid_used, /* Used on 32-bit */
.vector_allocation_domain = flat_vector_allocation_domain,
.init_apic_ldr = xen_noop, /* setup_local_APIC calls it */ .init_apic_ldr = xen_noop, /* setup_local_APIC calls it */
.ioapic_phys_id_map = default_ioapic_phys_id_map, /* Used on 32-bit */ .ioapic_phys_id_map = default_ioapic_phys_id_map, /* Used on 32-bit */
...@@ -179,7 +177,7 @@ static struct apic xen_pv_apic = { ...@@ -179,7 +177,7 @@ static struct apic xen_pv_apic = {
.get_apic_id = xen_get_apic_id, .get_apic_id = xen_get_apic_id,
.set_apic_id = xen_set_apic_id, /* Can be NULL on 32-bit. */ .set_apic_id = xen_set_apic_id, /* Can be NULL on 32-bit. */
.cpu_mask_to_apicid = flat_cpu_mask_to_apicid, .calc_dest_apicid = apic_flat_calc_apicid,
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
.send_IPI_mask = xen_send_IPI_mask, .send_IPI_mask = xen_send_IPI_mask,
......
...@@ -1230,6 +1230,7 @@ asmlinkage __visible void __init xen_start_kernel(void) ...@@ -1230,6 +1230,7 @@ asmlinkage __visible void __init xen_start_kernel(void)
x86_platform.get_nmi_reason = xen_get_nmi_reason; x86_platform.get_nmi_reason = xen_get_nmi_reason;
x86_init.resources.memory_setup = xen_memory_setup; x86_init.resources.memory_setup = xen_memory_setup;
x86_init.irqs.intr_mode_init = x86_init_noop;
x86_init.oem.arch_setup = xen_arch_setup; x86_init.oem.arch_setup = xen_arch_setup;
x86_init.oem.banner = xen_banner; x86_init.oem.banner = xen_banner;
......
...@@ -4173,16 +4173,25 @@ static void irq_remapping_free(struct irq_domain *domain, unsigned int virq, ...@@ -4173,16 +4173,25 @@ static void irq_remapping_free(struct irq_domain *domain, unsigned int virq,
irq_domain_free_irqs_common(domain, virq, nr_irqs); irq_domain_free_irqs_common(domain, virq, nr_irqs);
} }
static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu,
struct amd_ir_data *ir_data,
struct irq_2_irte *irte_info,
struct irq_cfg *cfg);
static int irq_remapping_activate(struct irq_domain *domain, static int irq_remapping_activate(struct irq_domain *domain,
struct irq_data *irq_data, bool early) struct irq_data *irq_data, bool early)
{ {
struct amd_ir_data *data = irq_data->chip_data; struct amd_ir_data *data = irq_data->chip_data;
struct irq_2_irte *irte_info = &data->irq_2_irte; struct irq_2_irte *irte_info = &data->irq_2_irte;
struct amd_iommu *iommu = amd_iommu_rlookup_table[irte_info->devid]; struct amd_iommu *iommu = amd_iommu_rlookup_table[irte_info->devid];
struct irq_cfg *cfg = irqd_cfg(irq_data);
if (iommu) if (!iommu)
iommu->irte_ops->activate(data->entry, irte_info->devid, return 0;
irte_info->index);
iommu->irte_ops->activate(data->entry, irte_info->devid,
irte_info->index);
amd_ir_update_irte(irq_data, iommu, data, irte_info, cfg);
return 0; return 0;
} }
...@@ -4270,6 +4279,22 @@ static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) ...@@ -4270,6 +4279,22 @@ static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info)
return modify_irte_ga(irte_info->devid, irte_info->index, irte, ir_data); return modify_irte_ga(irte_info->devid, irte_info->index, irte, ir_data);
} }
static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu,
struct amd_ir_data *ir_data,
struct irq_2_irte *irte_info,
struct irq_cfg *cfg)
{
/*
* Atomically updates the IRTE with the new destination, vector
* and flushes the interrupt entry cache.
*/
iommu->irte_ops->set_affinity(ir_data->entry, irte_info->devid,
irte_info->index, cfg->vector,
cfg->dest_apicid);
}
static int amd_ir_set_affinity(struct irq_data *data, static int amd_ir_set_affinity(struct irq_data *data,
const struct cpumask *mask, bool force) const struct cpumask *mask, bool force)
{ {
...@@ -4287,13 +4312,7 @@ static int amd_ir_set_affinity(struct irq_data *data, ...@@ -4287,13 +4312,7 @@ static int amd_ir_set_affinity(struct irq_data *data,
if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE) if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
return ret; return ret;
/* amd_ir_update_irte(data, iommu, ir_data, irte_info, cfg);
* Atomically updates the IRTE with the new destination, vector
* and flushes the interrupt entry cache.
*/
iommu->irte_ops->set_affinity(ir_data->entry, irte_info->devid,
irte_info->index, cfg->vector, cfg->dest_apicid);
/* /*
* After this point, all the interrupts will start arriving * After this point, all the interrupts will start arriving
* at the new destination. So, time to cleanup the previous * at the new destination. So, time to cleanup the previous
......
...@@ -1122,6 +1122,24 @@ struct irq_remap_ops intel_irq_remap_ops = { ...@@ -1122,6 +1122,24 @@ struct irq_remap_ops intel_irq_remap_ops = {
.get_irq_domain = intel_get_irq_domain, .get_irq_domain = intel_get_irq_domain,
}; };
static void intel_ir_reconfigure_irte(struct irq_data *irqd, bool force)
{
struct intel_ir_data *ir_data = irqd->chip_data;
struct irte *irte = &ir_data->irte_entry;
struct irq_cfg *cfg = irqd_cfg(irqd);
/*
* Atomically updates the IRTE with the new destination, vector
* and flushes the interrupt entry cache.
*/
irte->vector = cfg->vector;
irte->dest_id = IRTE_DEST(cfg->dest_apicid);
/* Update the hardware only if the interrupt is in remapped mode. */
if (!force || ir_data->irq_2_iommu.mode == IRQ_REMAPPING)
modify_irte(&ir_data->irq_2_iommu, irte);
}
/* /*
* Migrate the IO-APIC irq in the presence of intr-remapping. * Migrate the IO-APIC irq in the presence of intr-remapping.
* *
...@@ -1140,27 +1158,15 @@ static int ...@@ -1140,27 +1158,15 @@ static int
intel_ir_set_affinity(struct irq_data *data, const struct cpumask *mask, intel_ir_set_affinity(struct irq_data *data, const struct cpumask *mask,
bool force) bool force)
{ {
struct intel_ir_data *ir_data = data->chip_data;
struct irte *irte = &ir_data->irte_entry;
struct irq_cfg *cfg = irqd_cfg(data);
struct irq_data *parent = data->parent_data; struct irq_data *parent = data->parent_data;
struct irq_cfg *cfg = irqd_cfg(data);
int ret; int ret;
ret = parent->chip->irq_set_affinity(parent, mask, force); ret = parent->chip->irq_set_affinity(parent, mask, force);
if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE) if (ret < 0 || ret == IRQ_SET_MASK_OK_DONE)
return ret; return ret;
/* intel_ir_reconfigure_irte(data, false);
* Atomically updates the IRTE with the new destination, vector
* and flushes the interrupt entry cache.
*/
irte->vector = cfg->vector;
irte->dest_id = IRTE_DEST(cfg->dest_apicid);
/* Update the hardware only if the interrupt is in remapped mode. */
if (ir_data->irq_2_iommu.mode == IRQ_REMAPPING)
modify_irte(&ir_data->irq_2_iommu, irte);
/* /*
* After this point, all the interrupts will start arriving * After this point, all the interrupts will start arriving
* at the new destination. So, time to cleanup the previous * at the new destination. So, time to cleanup the previous
...@@ -1393,9 +1399,7 @@ static void intel_irq_remapping_free(struct irq_domain *domain, ...@@ -1393,9 +1399,7 @@ static void intel_irq_remapping_free(struct irq_domain *domain,
static int intel_irq_remapping_activate(struct irq_domain *domain, static int intel_irq_remapping_activate(struct irq_domain *domain,
struct irq_data *irq_data, bool early) struct irq_data *irq_data, bool early)
{ {
struct intel_ir_data *data = irq_data->chip_data; intel_ir_reconfigure_irte(irq_data, true);
modify_irte(&data->irq_2_iommu, &data->irte_entry);
return 0; return 0;
} }
......
...@@ -1441,6 +1441,8 @@ struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode, ...@@ -1441,6 +1441,8 @@ struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode,
pci_msi_domain_update_chip_ops(info); pci_msi_domain_update_chip_ops(info);
info->flags |= MSI_FLAG_ACTIVATE_EARLY; info->flags |= MSI_FLAG_ACTIVATE_EARLY;
if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE))
info->flags |= MSI_FLAG_MUST_REACTIVATE;
domain = msi_create_irq_domain(fwnode, info, parent); domain = msi_create_irq_domain(fwnode, info, parent);
if (!domain) if (!domain)
......
...@@ -666,12 +666,12 @@ asmlinkage __visible void __init start_kernel(void) ...@@ -666,12 +666,12 @@ asmlinkage __visible void __init start_kernel(void)
debug_objects_mem_init(); debug_objects_mem_init();
setup_per_cpu_pageset(); setup_per_cpu_pageset();
numa_policy_init(); numa_policy_init();
acpi_early_init();
if (late_time_init) if (late_time_init)
late_time_init(); late_time_init();
calibrate_delay(); calibrate_delay();
pidmap_init(); pidmap_init();
anon_vma_init(); anon_vma_init();
acpi_early_init();
#ifdef CONFIG_X86 #ifdef CONFIG_X86
if (efi_enabled(EFI_RUNTIME_SERVICES)) if (efi_enabled(EFI_RUNTIME_SERVICES))
efi_enter_virtual_mode(); efi_enter_virtual_mode();
......
...@@ -100,6 +100,9 @@ config IRQ_TIMINGS ...@@ -100,6 +100,9 @@ config IRQ_TIMINGS
config GENERIC_IRQ_MATRIX_ALLOCATOR config GENERIC_IRQ_MATRIX_ALLOCATOR
bool bool
config GENERIC_IRQ_RESERVATION_MODE
bool
config IRQ_DOMAIN_DEBUG config IRQ_DOMAIN_DEBUG
bool "Expose hardware/virtual IRQ mapping via debugfs" bool "Expose hardware/virtual IRQ mapping via debugfs"
depends on IRQ_DOMAIN && DEBUG_FS depends on IRQ_DOMAIN && DEBUG_FS
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment