Skip to content

L
linux
Commit 3f583bc2 authored by Linus Torvalds's avatar Linus Torvalds
Browse files
Options

Merge tag 'iommu-updates-v3.15' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu 

Pull IOMMU upates from Joerg Roedel:
 "This time a few more updates queued up.

   - Rework VT-d code to support ACPI devices

   - Improvements for memory and PCI hotplug support in the VT-d driver

   - Device-tree support for OMAP IOMMU

   - Convert OMAP IOMMU to use devm_* interfaces

   - Fixed PASID support for AMD IOMMU

   - Other random cleanups and fixes for OMAP, ARM-SMMU and SHMOBILE
     IOMMU

  Most of the changes are in the VT-d driver because some rework was
  necessary for better hotplug and ACPI device support"

* tag 'iommu-updates-v3.15' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu: (75 commits)
  iommu/vt-d: Fix error handling in ANDD processing
  iommu/vt-d: returning free pointer in get_domain_for_dev()
  iommu/vt-d: Only call dmar_acpi_dev_scope_init() if DRHD units present
  iommu/vt-d: Check for NULL pointer in dmar_acpi_dev_scope_init()
  iommu/amd: Fix logic to determine and checking max PASID
  iommu/vt-d: Include ACPI devices in iommu=pt
  iommu/vt-d: Finally enable translation for non-PCI devices
  iommu/vt-d: Remove to_pci_dev() in intel_map_page()
  iommu/vt-d: Remove pdev from intel_iommu_attach_device()
  iommu/vt-d: Remove pdev from iommu_no_mapping()
  iommu/vt-d: Make domain_add_dev_info() take struct device
  iommu/vt-d: Make domain_remove_one_dev_info() take struct device
  iommu/vt-d: Rename 'hwdev' variables to 'dev' now that that's the norm
  iommu/vt-d: Remove some pointless to_pci_dev() calls
  iommu/vt-d: Make get_valid_domain_for_dev() take struct device
  iommu/vt-d: Make iommu_should_identity_map() take struct device
  iommu/vt-d: Handle RMRRs for non-PCI devices
  iommu/vt-d: Make get_domain_for_dev() take struct device
  iommu/vt-d: Make domain_context_mapp{ed,ing}() take struct device
  iommu/vt-d: Make device_to_iommu() cope with non-PCI devices
  ...
parents 3e76b749 e172b812
Hide whitespace changes
Inline Side-by-side
Showing with 1754 additions and 990 deletions
Documentation/devicetree/bindings/iommu/arm,smmu.txt
View file @ 3f583bc2
......@@ -48,6 +48,12 @@ conditions.
from the mmu-masters towards memory) node for this
SMMU.
- calxeda,smmu-secure-config-access : Enable proper handling of buggy
implementations that always use secure access to
SMMU configuration registers. In this case non-secure
aliases of secure registers have to be used during
SMMU configuration.
Example:
smmu {
......
Documentation/devicetree/bindings/iommu/ti,omap-iommu.txt 0 → 100644
View file @ 3f583bc2
OMAP2+ IOMMU
Required properties:
- compatible : Should be one of,
"ti,omap2-iommu" for OMAP2/OMAP3 IOMMU instances
"ti,omap4-iommu" for OMAP4/OMAP5 IOMMU instances
"ti,dra7-iommu" for DRA7xx IOMMU instances
- ti,hwmods : Name of the hwmod associated with the IOMMU instance
- reg : Address space for the configuration registers
- interrupts : Interrupt specifier for the IOMMU instance
Optional properties:
- ti,#tlb-entries : Number of entries in the translation look-aside buffer.
Should be either 8 or 32 (default: 32)
- ti,iommu-bus-err-back : Indicates the IOMMU instance supports throwing
back a bus error response on MMU faults.
Example:
/* OMAP3 ISP MMU */
mmu_isp: mmu@480bd400 {
compatible = "ti,omap2-iommu";
reg = <0x480bd400 0x80>;
interrupts = <24>;
ti,hwmods = "mmu_isp";
ti,#tlb-entries = <8>;
};
arch/arm/mach-omap2/omap-iommu.c
View file @ 3f583bc2
......@@ -10,6 +10,7 @@
* published by the Free Software Foundation.
*/
#include <linux/of.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/err.h>
......@@ -58,6 +59,10 @@ static int __init omap_iommu_dev_init(struct omap_hwmod *oh, void *unused)
static int __init omap_iommu_init(void)
{
/* If dtb is there, the devices will be created dynamically */
if (of_have_populated_dt())
return -ENODEV;
return omap_hwmod_for_each_by_class("mmu", omap_iommu_dev_init, NULL);
}
/* must be ready before omap3isp is probed */
......
drivers/iommu/Kconfig
View file @ 3f583bc2
......@@ -207,7 +207,7 @@ config SHMOBILE_IOMMU
bool "IOMMU for Renesas IPMMU/IPMMUI"
default n
depends on ARM
depends on SH_MOBILE || COMPILE_TEST
depends on ARCH_SHMOBILE || COMPILE_TEST
select IOMMU_API
select ARM_DMA_USE_IOMMU
select SHMOBILE_IPMMU
......
drivers/iommu/amd_iommu.c
View file @ 3f583bc2
......@@ -963,7 +963,7 @@ static void build_inv_iommu_pasid(struct iommu_cmd *cmd, u16 domid, int pasid,
address &= ~(0xfffULL);
cmd->data[0] = pasid & PASID_MASK;
cmd->data[0] = pasid;
cmd->data[1] = domid;
cmd->data[2] = lower_32_bits(address);
cmd->data[3] = upper_32_bits(address);
......@@ -982,10 +982,10 @@ static void build_inv_iotlb_pasid(struct iommu_cmd *cmd, u16 devid, int pasid,
address &= ~(0xfffULL);
cmd->data[0] = devid;
cmd->data[0] |= (pasid & 0xff) << 16;
cmd->data[0] |= ((pasid >> 8) & 0xff) << 16;
cmd->data[0] |= (qdep & 0xff) << 24;
cmd->data[1] = devid;
cmd->data[1] |= ((pasid >> 8) & 0xfff) << 16;
cmd->data[1] |= (pasid & 0xff) << 16;
cmd->data[2] = lower_32_bits(address);
cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK;
cmd->data[3] = upper_32_bits(address);
......@@ -1001,7 +1001,7 @@ static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, int pasid,
cmd->data[0] = devid;
if (gn) {
cmd->data[1] = pasid & PASID_MASK;
cmd->data[1] = pasid;
cmd->data[2] = CMD_INV_IOMMU_PAGES_GN_MASK;
}
cmd->data[3] = tag & 0x1ff;
......
drivers/iommu/amd_iommu_init.c
View file @ 3f583bc2
......@@ -150,7 +150,7 @@ int amd_iommus_present;
bool amd_iommu_np_cache __read_mostly;
bool amd_iommu_iotlb_sup __read_mostly = true;
u32 amd_iommu_max_pasids __read_mostly = ~0;
u32 amd_iommu_max_pasid __read_mostly = ~0;
bool amd_iommu_v2_present __read_mostly;
bool amd_iommu_pc_present __read_mostly;
......@@ -1231,14 +1231,16 @@ static int iommu_init_pci(struct amd_iommu *iommu)
if (iommu_feature(iommu, FEATURE_GT)) {
int glxval;
u32 pasids;
u64 shift;
u32 max_pasid;
u64 pasmax;
shift = iommu->features & FEATURE_PASID_MASK;
shift >>= FEATURE_PASID_SHIFT;
pasids = (1 << shift);
pasmax = iommu->features & FEATURE_PASID_MASK;
pasmax >>= FEATURE_PASID_SHIFT;
max_pasid = (1 << (pasmax + 1)) - 1;
amd_iommu_max_pasids = min(amd_iommu_max_pasids, pasids);
amd_iommu_max_pasid = min(amd_iommu_max_pasid, max_pasid);
BUG_ON(amd_iommu_max_pasid & ~PASID_MASK);
glxval = iommu->features & FEATURE_GLXVAL_MASK;
glxval >>= FEATURE_GLXVAL_SHIFT;
......
drivers/iommu/amd_iommu_types.h
View file @ 3f583bc2
......@@ -99,7 +99,12 @@
#define FEATURE_GLXVAL_SHIFT 14
#define FEATURE_GLXVAL_MASK (0x03ULL << FEATURE_GLXVAL_SHIFT)
#define PASID_MASK 0x000fffff
/* Note:
* The current driver only support 16-bit PASID.
* Currently, hardware only implement upto 16-bit PASID
* even though the spec says it could have upto 20 bits.
*/
#define PASID_MASK 0x0000ffff
/* MMIO status bits */
#define MMIO_STATUS_EVT_INT_MASK (1 << 1)
......@@ -697,8 +702,8 @@ extern unsigned long *amd_iommu_pd_alloc_bitmap;
*/
extern u32 amd_iommu_unmap_flush;
/* Smallest number of PASIDs supported by any IOMMU in the system */
extern u32 amd_iommu_max_pasids;
/* Smallest max PASID supported by any IOMMU in the system */
extern u32 amd_iommu_max_pasid;
extern bool amd_iommu_v2_present;
......
drivers/iommu/arm-smmu.c
View file @ 3f583bc2
......@@ -48,7 +48,7 @@
#include <asm/pgalloc.h>
/* Maximum number of stream IDs assigned to a single device */
#define MAX_MASTER_STREAMIDS 8
#define MAX_MASTER_STREAMIDS MAX_PHANDLE_ARGS
/* Maximum number of context banks per SMMU */
#define ARM_SMMU_MAX_CBS 128
......@@ -60,6 +60,16 @@
#define ARM_SMMU_GR0(smmu) ((smmu)->base)
#define ARM_SMMU_GR1(smmu) ((smmu)->base + (smmu)->pagesize)
/*
* SMMU global address space with conditional offset to access secure
* aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448,
* nsGFSYNR0: 0x450)
*/
#define ARM_SMMU_GR0_NS(smmu) \
((smmu)->base + \
((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \
? 0x400 : 0))
/* Page table bits */
#define ARM_SMMU_PTE_XN (((pteval_t)3) << 53)
#define ARM_SMMU_PTE_CONT (((pteval_t)1) << 52)
......@@ -351,6 +361,9 @@ struct arm_smmu_device {
#define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
#define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
u32 features;
#define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0)
u32 options;
int version;
u32 num_context_banks;
......@@ -401,6 +414,29 @@ struct arm_smmu_domain {
static DEFINE_SPINLOCK(arm_smmu_devices_lock);
static LIST_HEAD(arm_smmu_devices);
struct arm_smmu_option_prop {
u32 opt;
const char *prop;
};
static struct arm_smmu_option_prop arm_smmu_options [] = {
{ ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" },
{ 0, NULL},
};
static void parse_driver_options(struct arm_smmu_device *smmu)
{
int i = 0;
do {
if (of_property_read_bool(smmu->dev->of_node,
arm_smmu_options[i].prop)) {
smmu->options |= arm_smmu_options[i].opt;
dev_notice(smmu->dev, "option %s\n",
arm_smmu_options[i].prop);
}
} while (arm_smmu_options[++i].opt);
}
static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
struct device_node *dev_node)
{
......@@ -614,16 +650,16 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
{
u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
struct arm_smmu_device *smmu = dev;
void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
if (!gfsr)
return IRQ_NONE;
gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
if (!gfsr)
return IRQ_NONE;
dev_err_ratelimited(smmu->dev,
"Unexpected global fault, this could be serious\n");
dev_err_ratelimited(smmu->dev,
......@@ -642,7 +678,7 @@ static void arm_smmu_flush_pgtable(struct arm_smmu_device *smmu, void *addr,
/* Ensure new page tables are visible to the hardware walker */
if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) {
dsb();
dsb(ishst);
} else {
/*
* If the SMMU can't walk tables in the CPU caches, treat them
......@@ -990,9 +1026,8 @@ static void arm_smmu_free_pgtables(struct arm_smmu_domain *smmu_domain)
/*
* Recursively free the page tables for this domain. We don't
* care about speculative TLB filling, because the TLB will be
* nuked next time this context bank is re-allocated and no devices
* currently map to these tables.
* care about speculative TLB filling because the tables should
* not be active in any context bank at this point (SCTLR.M is 0).
*/
pgd = pgd_base;
for (i = 0; i < PTRS_PER_PGD; ++i) {
......@@ -1218,7 +1253,7 @@ static bool arm_smmu_pte_is_contiguous_range(unsigned long addr,
static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd,
unsigned long addr, unsigned long end,
unsigned long pfn, int flags, int stage)
unsigned long pfn, int prot, int stage)
{
pte_t *pte, *start;
pteval_t pteval = ARM_SMMU_PTE_PAGE | ARM_SMMU_PTE_AF | ARM_SMMU_PTE_XN;
......@@ -1240,28 +1275,28 @@ static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd,
if (stage == 1) {
pteval |= ARM_SMMU_PTE_AP_UNPRIV | ARM_SMMU_PTE_nG;
if (!(flags & IOMMU_WRITE) && (flags & IOMMU_READ))
if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ))
pteval |= ARM_SMMU_PTE_AP_RDONLY;
if (flags & IOMMU_CACHE)
if (prot & IOMMU_CACHE)
pteval |= (MAIR_ATTR_IDX_CACHE <<
ARM_SMMU_PTE_ATTRINDX_SHIFT);
} else {
pteval |= ARM_SMMU_PTE_HAP_FAULT;
if (flags & IOMMU_READ)
if (prot & IOMMU_READ)
pteval |= ARM_SMMU_PTE_HAP_READ;
if (flags & IOMMU_WRITE)
if (prot & IOMMU_WRITE)
pteval |= ARM_SMMU_PTE_HAP_WRITE;
if (flags & IOMMU_CACHE)
if (prot & IOMMU_CACHE)
pteval |= ARM_SMMU_PTE_MEMATTR_OIWB;
else
pteval |= ARM_SMMU_PTE_MEMATTR_NC;
}
/* If no access, create a faulting entry to avoid TLB fills */
if (flags & IOMMU_EXEC)
if (prot & IOMMU_EXEC)
pteval &= ~ARM_SMMU_PTE_XN;
else if (!(flags & (IOMMU_READ | IOMMU_WRITE)))
else if (!(prot & (IOMMU_READ | IOMMU_WRITE)))
pteval &= ~ARM_SMMU_PTE_PAGE;
pteval |= ARM_SMMU_PTE_SH_IS;
......@@ -1323,7 +1358,7 @@ static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd,
static int arm_smmu_alloc_init_pmd(struct arm_smmu_device *smmu, pud_t *pud,
unsigned long addr, unsigned long end,
phys_addr_t phys, int flags, int stage)
phys_addr_t phys, int prot, int stage)
{
int ret;
pmd_t *pmd;
......@@ -1347,7 +1382,7 @@ static int arm_smmu_alloc_init_pmd(struct arm_smmu_device *smmu, pud_t *pud,
do {
next = pmd_addr_end(addr, end);
ret = arm_smmu_alloc_init_pte(smmu, pmd, addr, end, pfn,
flags, stage);
prot, stage);
phys += next - addr;
} while (pmd++, addr = next, addr < end);
......@@ -1356,7 +1391,7 @@ static int arm_smmu_alloc_init_pmd(struct arm_smmu_device *smmu, pud_t *pud,
static int arm_smmu_alloc_init_pud(struct arm_smmu_device *smmu, pgd_t *pgd,
unsigned long addr, unsigned long end,
phys_addr_t phys, int flags, int stage)
phys_addr_t phys, int prot, int stage)
{
int ret = 0;
pud_t *pud;
......@@ -1380,7 +1415,7 @@ static int arm_smmu_alloc_init_pud(struct arm_smmu_device *smmu, pgd_t *pgd,
do {
next = pud_addr_end(addr, end);
ret = arm_smmu_alloc_init_pmd(smmu, pud, addr, next, phys,
flags, stage);
prot, stage);
phys += next - addr;
} while (pud++, addr = next, addr < end);
......@@ -1389,7 +1424,7 @@ static int arm_smmu_alloc_init_pud(struct arm_smmu_device *smmu, pgd_t *pgd,
static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain,
unsigned long iova, phys_addr_t paddr,
size_t size, int flags)
size_t size, int prot)
{
int ret, stage;
unsigned long end;
......@@ -1397,7 +1432,7 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain,
struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg;
pgd_t *pgd = root_cfg->pgd;
struct arm_smmu_device *smmu = root_cfg->smmu;
unsigned long irqflags;
unsigned long flags;
if (root_cfg->cbar == CBAR_TYPE_S2_TRANS) {
stage = 2;
......@@ -1420,14 +1455,14 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain,
if (paddr & ~output_mask)
return -ERANGE;
spin_lock_irqsave(&smmu_domain->lock, irqflags);
spin_lock_irqsave(&smmu_domain->lock, flags);
pgd += pgd_index(iova);
end = iova + size;
do {
unsigned long next = pgd_addr_end(iova, end);
ret = arm_smmu_alloc_init_pud(smmu, pgd, iova, next, paddr,
flags, stage);
prot, stage);
if (ret)
goto out_unlock;
......@@ -1436,13 +1471,13 @@ static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain,
} while (pgd++, iova != end);
out_unlock:
spin_unlock_irqrestore(&smmu_domain->lock, irqflags);
spin_unlock_irqrestore(&smmu_domain->lock, flags);
return ret;
}
static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int flags)
phys_addr_t paddr, size_t size, int prot)
{
struct arm_smmu_domain *smmu_domain = domain->priv;
......@@ -1453,7 +1488,7 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
if ((phys_addr_t)iova & ~smmu_domain->output_mask)
return -ERANGE;
return arm_smmu_handle_mapping(smmu_domain, iova, paddr, size, flags);
return arm_smmu_handle_mapping(smmu_domain, iova, paddr, size, prot);
}
static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
......@@ -1597,9 +1632,9 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
int i = 0;
u32 reg;
/* Clear Global FSR */
reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
writel(reg, gr0_base + ARM_SMMU_GR0_sGFSR);
/* clear global FSR */
reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
/* Mark all SMRn as invalid and all S2CRn as bypass */
for (i = 0; i < smmu->num_mapping_groups; ++i) {
......@@ -1619,7 +1654,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sCR0);
reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
/* Enable fault reporting */
reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
......@@ -1638,7 +1673,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
/* Push the button */
arm_smmu_tlb_sync(smmu);
writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sCR0);
writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
}
static int arm_smmu_id_size_to_bits(int size)
......@@ -1885,6 +1920,8 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
if (err)
goto out_put_parent;
parse_driver_options(smmu);
if (smmu->version > 1 &&
smmu->num_context_banks != smmu->num_context_irqs) {
dev_err(dev,
......@@ -1969,7 +2006,7 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
free_irq(smmu->irqs[i], smmu);
/* Turn the thing off */
writel_relaxed(sCR0_CLIENTPD, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_sCR0);
writel(sCR0_CLIENTPD,ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
return 0;
}
......
drivers/iommu/dmar.c
View file @ 3f583bc2
......@@ -43,14 +43,24 @@
#include "irq_remapping.h"
/* No locks are needed as DMA remapping hardware unit
* list is constructed at boot time and hotplug of
* these units are not supported by the architecture.
/*
* Assumptions:
* 1) The hotplug framework guarentees that DMAR unit will be hot-added
* before IO devices managed by that unit.
* 2) The hotplug framework guarantees that DMAR unit will be hot-removed
* after IO devices managed by that unit.
* 3) Hotplug events are rare.
*
* Locking rules for DMA and interrupt remapping related global data structures:
* 1) Use dmar_global_lock in process context
* 2) Use RCU in interrupt context
*/
DECLARE_RWSEM(dmar_global_lock);
LIST_HEAD(dmar_drhd_units);
struct acpi_table_header * __initdata dmar_tbl;
static acpi_size dmar_tbl_size;
static int dmar_dev_scope_status = 1;
static int alloc_iommu(struct dmar_drhd_unit *drhd);
static void free_iommu(struct intel_iommu *iommu);
......@@ -62,73 +72,20 @@ static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
* the very end.
*/
if (drhd->include_all)
list_add_tail(&drhd->list, &dmar_drhd_units);
list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
else
list_add(&drhd->list, &dmar_drhd_units);
list_add_rcu(&drhd->list, &dmar_drhd_units);
}
static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
struct pci_dev **dev, u16 segment)
{
struct pci_bus *bus;
struct pci_dev *pdev = NULL;
struct acpi_dmar_pci_path *path;
int count;
bus = pci_find_bus(segment, scope->bus);
path = (struct acpi_dmar_pci_path *)(scope + 1);
count = (scope->length - sizeof(struct acpi_dmar_device_scope))
/ sizeof(struct acpi_dmar_pci_path);
while (count) {
if (pdev)
pci_dev_put(pdev);
/*
* Some BIOSes list non-exist devices in DMAR table, just
* ignore it
*/
if (!bus) {
pr_warn("Device scope bus [%d] not found\n", scope->bus);
break;
}
pdev = pci_get_slot(bus, PCI_DEVFN(path->device, path->function));
if (!pdev) {
/* warning will be printed below */
break;
}
path ++;
count --;
bus = pdev->subordinate;
}
if (!pdev) {
pr_warn("Device scope device [%04x:%02x:%02x.%02x] not found\n",
segment, scope->bus, path->device, path->function);
return 0;
}
if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
pdev->subordinate) || (scope->entry_type == \
ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
pci_dev_put(pdev);
pr_warn("Device scope type does not match for %s\n",
pci_name(pdev));
return -EINVAL;
}
*dev = pdev;
return 0;
}
int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
struct pci_dev ***devices, u16 segment)
void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
{
struct acpi_dmar_device_scope *scope;
void * tmp = start;
int index;
int ret;
*cnt = 0;
while (start < end) {
scope = start;
if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ACPI ||
scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
(*cnt)++;
else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
......@@ -138,43 +95,236 @@ int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
start += scope->length;
}
if (*cnt == 0)
return 0;
return NULL;
*devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
if (!*devices)
return -ENOMEM;
return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
}
start = tmp;
index = 0;
while (start < end) {
void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
{
int i;
struct device *tmp_dev;
if (*devices && *cnt) {
for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
put_device(tmp_dev);
kfree(*devices);
}
*devices = NULL;
*cnt = 0;
}
/* Optimize out kzalloc()/kfree() for normal cases */
static char dmar_pci_notify_info_buf[64];
static struct dmar_pci_notify_info *
dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
{
int level = 0;
size_t size;
struct pci_dev *tmp;
struct dmar_pci_notify_info *info;
BUG_ON(dev->is_virtfn);
/* Only generate path[] for device addition event */
if (event == BUS_NOTIFY_ADD_DEVICE)
for (tmp = dev; tmp; tmp = tmp->bus->self)
level++;
size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path);
if (size <= sizeof(dmar_pci_notify_info_buf)) {
info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
} else {
info = kzalloc(size, GFP_KERNEL);
if (!info) {
pr_warn("Out of memory when allocating notify_info "
"for %s.\n", pci_name(dev));
if (dmar_dev_scope_status == 0)
dmar_dev_scope_status = -ENOMEM;
return NULL;
}
}
info->event = event;
info->dev = dev;
info->seg = pci_domain_nr(dev->bus);
info->level = level;
if (event == BUS_NOTIFY_ADD_DEVICE) {
for (tmp = dev, level--; tmp; tmp = tmp->bus->self) {
info->path[level].device = PCI_SLOT(tmp->devfn);
info->path[level].function = PCI_FUNC(tmp->devfn);
if (pci_is_root_bus(tmp->bus))
info->bus = tmp->bus->number;
}
}
return info;
}
static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
{
if ((void *)info != dmar_pci_notify_info_buf)
kfree(info);
}
static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
struct acpi_dmar_pci_path *path, int count)
{
int i;
if (info->bus != bus)
return false;
if (info->level != count)
return false;
for (i = 0; i < count; i++) {
if (path[i].device != info->path[i].device ||
path[i].function != info->path[i].function)
return false;
}
return true;
}
/* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
void *start, void*end, u16 segment,
struct dmar_dev_scope *devices,
int devices_cnt)
{
int i, level;
struct device *tmp, *dev = &info->dev->dev;
struct acpi_dmar_device_scope *scope;
struct acpi_dmar_pci_path *path;
if (segment != info->seg)
return 0;
for (; start < end; start += scope->length) {
scope = start;
if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
ret = dmar_parse_one_dev_scope(scope,
&(*devices)[index], segment);
if (ret) {
dmar_free_dev_scope(devices, cnt);
return ret;
}
index ++;
if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
continue;
path = (struct acpi_dmar_pci_path *)(scope + 1);
level = (scope->length - sizeof(*scope)) / sizeof(*path);
if (!dmar_match_pci_path(info, scope->bus, path, level))
continue;
if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^
(info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) {
pr_warn("Device scope type does not match for %s\n",
pci_name(info->dev));
return -EINVAL;
}
start += scope->length;
for_each_dev_scope(devices, devices_cnt, i, tmp)
if (tmp == NULL) {
devices[i].bus = info->dev->bus->number;
devices[i].devfn = info->dev->devfn;
rcu_assign_pointer(devices[i].dev,
get_device(dev));
return 1;
}
BUG_ON(i >= devices_cnt);
}
return 0;
}
void dmar_free_dev_scope(struct pci_dev ***devices, int *cnt)
int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
struct dmar_dev_scope *devices, int count)
{
if (*devices && *cnt) {
while (--*cnt >= 0)
pci_dev_put((*devices)[*cnt]);
kfree(*devices);
*devices = NULL;
*cnt = 0;
int index;
struct device *tmp;
if (info->seg != segment)
return 0;
for_each_active_dev_scope(devices, count, index, tmp)
if (tmp == &info->dev->dev) {
rcu_assign_pointer(devices[index].dev, NULL);
synchronize_rcu();
put_device(tmp);
return 1;
}
return 0;
}
static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
{
int ret = 0;
struct dmar_drhd_unit *dmaru;
struct acpi_dmar_hardware_unit *drhd;
for_each_drhd_unit(dmaru) {
if (dmaru->include_all)
continue;
drhd = container_of(dmaru->hdr,
struct acpi_dmar_hardware_unit, header);
ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
((void *)drhd) + drhd->header.length,
dmaru->segment,
dmaru->devices, dmaru->devices_cnt);
if (ret != 0)
break;
}
if (ret >= 0)
ret = dmar_iommu_notify_scope_dev(info);
if (ret < 0 && dmar_dev_scope_status == 0)
dmar_dev_scope_status = ret;
return ret;
}
static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
{
struct dmar_drhd_unit *dmaru;
for_each_drhd_unit(dmaru)
if (dmar_remove_dev_scope(info, dmaru->segment,
dmaru->devices, dmaru->devices_cnt))
break;
dmar_iommu_notify_scope_dev(info);
}
static int dmar_pci_bus_notifier(struct notifier_block *nb,
unsigned long action, void *data)
{
struct pci_dev *pdev = to_pci_dev(data);
struct dmar_pci_notify_info *info;
/* Only care about add/remove events for physical functions */
if (pdev->is_virtfn)
return NOTIFY_DONE;
if (action != BUS_NOTIFY_ADD_DEVICE && action != BUS_NOTIFY_DEL_DEVICE)
return NOTIFY_DONE;
info = dmar_alloc_pci_notify_info(pdev, action);
if (!info)
return NOTIFY_DONE;
down_write(&dmar_global_lock);
if (action == BUS_NOTIFY_ADD_DEVICE)
dmar_pci_bus_add_dev(info);
else if (action == BUS_NOTIFY_DEL_DEVICE)
dmar_pci_bus_del_dev(info);
up_write(&dmar_global_lock);
dmar_free_pci_notify_info(info);
return NOTIFY_OK;
}
static struct notifier_block dmar_pci_bus_nb = {
.notifier_call = dmar_pci_bus_notifier,
.priority = INT_MIN,
};
/**
* dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
* structure which uniquely represent one DMA remapping hardware unit
......@@ -196,9 +346,18 @@ dmar_parse_one_drhd(struct acpi_dmar_header *header)
dmaru->reg_base_addr = drhd->address;
dmaru->segment = drhd->segment;
dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
((void *)drhd) + drhd->header.length,
&dmaru->devices_cnt);
if (dmaru->devices_cnt && dmaru->devices == NULL) {
kfree(dmaru);
return -ENOMEM;
}
ret = alloc_iommu(dmaru);
if (ret) {
dmar_free_dev_scope(&dmaru->devices,
&dmaru->devices_cnt);
kfree(dmaru);
return ret;
}
......@@ -215,19 +374,24 @@ static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
kfree(dmaru);
}
static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
static int __init dmar_parse_one_andd(struct acpi_dmar_header *header)
{
struct acpi_dmar_hardware_unit *drhd;
drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
if (dmaru->include_all)
return 0;
struct acpi_dmar_andd *andd = (void *)header;
/* Check for NUL termination within the designated length */
if (strnlen(andd->object_name, header->length - 8) == header->length - 8) {
WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
"Your BIOS is broken; ANDD object name is not NUL-terminated\n"
"BIOS vendor: %s; Ver: %s; Product Version: %s\n",
dmi_get_system_info(DMI_BIOS_VENDOR),
dmi_get_system_info(DMI_BIOS_VERSION),
dmi_get_system_info(DMI_PRODUCT_VERSION));
return -EINVAL;
}
pr_info("ANDD device: %x name: %s\n", andd->device_number,
andd->object_name);
return dmar_parse_dev_scope((void *)(drhd + 1),
((void *)drhd) + drhd->header.length,
&dmaru->devices_cnt, &dmaru->devices,
drhd->segment);
return 0;
}
#ifdef CONFIG_ACPI_NUMA
......@@ -293,6 +457,10 @@ dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
(unsigned long long)rhsa->base_address,
rhsa->proximity_domain);
break;
case ACPI_DMAR_TYPE_ANDD:
/* We don't print this here because we need to sanity-check
it first. So print it in dmar_parse_one_andd() instead. */
break;
}
}
......@@ -378,6 +546,9 @@ parse_dmar_table(void)
ret = dmar_parse_one_rhsa(entry_header);
#endif
break;
case ACPI_DMAR_TYPE_ANDD:
ret = dmar_parse_one_andd(entry_header);
break;
default:
pr_warn("Unknown DMAR structure type %d\n",
entry_header->type);
......@@ -394,14 +565,15 @@ parse_dmar_table(void)
return ret;
}
static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
struct pci_dev *dev)
static int dmar_pci_device_match(struct dmar_dev_scope devices[],
int cnt, struct pci_dev *dev)
{
int index;
struct device *tmp;
while (dev) {
for (index = 0; index < cnt; index++)
if (dev == devices[index])
for_each_active_dev_scope(devices, cnt, index, tmp)
if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
return 1;
/* Check our parent */
......@@ -414,11 +586,12 @@ static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
struct dmar_drhd_unit *
dmar_find_matched_drhd_unit(struct pci_dev *dev)
{
struct dmar_drhd_unit *dmaru = NULL;
struct dmar_drhd_unit *dmaru;
struct acpi_dmar_hardware_unit *drhd;
dev = pci_physfn(dev);
rcu_read_lock();
for_each_drhd_unit(dmaru) {
drhd = container_of(dmaru->hdr,
struct acpi_dmar_hardware_unit,
......@@ -426,44 +599,128 @@ dmar_find_matched_drhd_unit(struct pci_dev *dev)
if (dmaru->include_all &&
drhd->segment == pci_domain_nr(dev->bus))
return dmaru;
goto out;
if (dmar_pci_device_match(dmaru->devices,
dmaru->devices_cnt, dev))
return dmaru;
goto out;
}
dmaru = NULL;
out:
rcu_read_unlock();
return NULL;
return dmaru;
}
int __init dmar_dev_scope_init(void)
static void __init dmar_acpi_insert_dev_scope(u8 device_number,
struct acpi_device *adev)
{
static int dmar_dev_scope_initialized;
struct dmar_drhd_unit *drhd;
int ret = -ENODEV;
if (dmar_dev_scope_initialized)
return dmar_dev_scope_initialized;
struct dmar_drhd_unit *dmaru;
struct acpi_dmar_hardware_unit *drhd;
struct acpi_dmar_device_scope *scope;
struct device *tmp;
int i;
struct acpi_dmar_pci_path *path;
if (list_empty(&dmar_drhd_units))
goto fail;
for_each_drhd_unit(dmaru) {
drhd = container_of(dmaru->hdr,
struct acpi_dmar_hardware_unit,
header);
list_for_each_entry(drhd, &dmar_drhd_units, list) {
ret = dmar_parse_dev(drhd);
if (ret)
goto fail;
for (scope = (void *)(drhd + 1);
(unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
scope = ((void *)scope) + scope->length) {
if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ACPI)
continue;
if (scope->enumeration_id != device_number)
continue;
path = (void *)(scope + 1);
pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
dev_name(&adev->dev), dmaru->reg_base_addr,
scope->bus, path->device, path->function);
for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
if (tmp == NULL) {
dmaru->devices[i].bus = scope->bus;
dmaru->devices[i].devfn = PCI_DEVFN(path->device,
path->function);
rcu_assign_pointer(dmaru->devices[i].dev,
get_device(&adev->dev));
return;
}
BUG_ON(i >= dmaru->devices_cnt);
}
}
pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
device_number, dev_name(&adev->dev));
}
ret = dmar_parse_rmrr_atsr_dev();
if (ret)
goto fail;
static int __init dmar_acpi_dev_scope_init(void)
{
struct acpi_dmar_andd *andd;
if (dmar_tbl == NULL)
return -ENODEV;
dmar_dev_scope_initialized = 1;
for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
andd = ((void *)andd) + andd->header.length) {
if (andd->header.type == ACPI_DMAR_TYPE_ANDD) {
acpi_handle h;
struct acpi_device *adev;
if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
andd->object_name,
&h))) {
pr_err("Failed to find handle for ACPI object %s\n",
andd->object_name);
continue;
}
acpi_bus_get_device(h, &adev);
if (!adev) {
pr_err("Failed to get device for ACPI object %s\n",
andd->object_name);
continue;
}
dmar_acpi_insert_dev_scope(andd->device_number, adev);
}
}
return 0;
}
fail:
dmar_dev_scope_initialized = ret;
return ret;
int __init dmar_dev_scope_init(void)
{
struct pci_dev *dev = NULL;
struct dmar_pci_notify_info *info;
if (dmar_dev_scope_status != 1)
return dmar_dev_scope_status;
if (list_empty(&dmar_drhd_units)) {
dmar_dev_scope_status = -ENODEV;
} else {
dmar_dev_scope_status = 0;
dmar_acpi_dev_scope_init();
for_each_pci_dev(dev) {
if (dev->is_virtfn)
continue;
info = dmar_alloc_pci_notify_info(dev,
BUS_NOTIFY_ADD_DEVICE);
if (!info) {
return dmar_dev_scope_status;
} else {
dmar_pci_bus_add_dev(info);
dmar_free_pci_notify_info(info);
}
}
bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
}
return dmar_dev_scope_status;
}
......@@ -557,6 +814,7 @@ int __init detect_intel_iommu(void)
{
int ret;
down_write(&dmar_global_lock);
ret = dmar_table_detect();
if (ret)
ret = check_zero_address();
......@@ -574,6 +832,7 @@ int __init detect_intel_iommu(void)
}
early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size);
dmar_tbl = NULL;
up_write(&dmar_global_lock);
return ret ? 1 : -ENODEV;
}
......@@ -696,6 +955,7 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
}
iommu->agaw = agaw;
iommu->msagaw = msagaw;
iommu->segment = drhd->segment;
iommu->node = -1;
......@@ -1386,10 +1646,15 @@ static int __init dmar_free_unused_resources(void)
if (irq_remapping_enabled || intel_iommu_enabled)
return 0;
if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
down_write(&dmar_global_lock);
list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
list_del(&dmaru->list);
dmar_free_drhd(dmaru);
}
up_write(&dmar_global_lock);
return 0;
}
......
drivers/iommu/intel-iommu.c
View file @ 3f583bc2
/*
* Copyright (c) 2006, Intel Corporation.
* Copyright © 2006-2014 Intel Corporation.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
......@@ -10,15 +10,11 @@
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc., 59 Temple
* Place - Suite 330, Boston, MA 02111-1307 USA.
*
* Copyright (C) 2006-2008 Intel Corporation
* Author: Ashok Raj <ashok.raj@intel.com>
* Author: Shaohua Li <shaohua.li@intel.com>
* Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
* Author: Fenghua Yu <fenghua.yu@intel.com>
* Authors: David Woodhouse <dwmw2@infradead.org>,
* Ashok Raj <ashok.raj@intel.com>,
* Shaohua Li <shaohua.li@intel.com>,
* Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
* Fenghua Yu <fenghua.yu@intel.com>
*/
#include <linux/init.h>
......@@ -33,6 +29,7 @@
#include <linux/dmar.h>
#include <linux/dma-mapping.h>
#include <linux/mempool.h>
#include <linux/memory.h>
#include <linux/timer.h>
#include <linux/iova.h>
#include <linux/iommu.h>
......@@ -372,14 +369,36 @@ struct dmar_domain {
struct device_domain_info {
struct list_head link; /* link to domain siblings */
struct list_head global; /* link to global list */
int segment; /* PCI domain */
u8 bus; /* PCI bus number */
u8 devfn; /* PCI devfn number */
struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
struct intel_iommu *iommu; /* IOMMU used by this device */
struct dmar_domain *domain; /* pointer to domain */
};
struct dmar_rmrr_unit {
struct list_head list; /* list of rmrr units */
struct acpi_dmar_header *hdr; /* ACPI header */
u64 base_address; /* reserved base address*/
u64 end_address; /* reserved end address */
struct dmar_dev_scope *devices; /* target devices */
int devices_cnt; /* target device count */
};
struct dmar_atsr_unit {
struct list_head list; /* list of ATSR units */
struct acpi_dmar_header *hdr; /* ACPI header */
struct dmar_dev_scope *devices; /* target devices */
int devices_cnt; /* target device count */
u8 include_all:1; /* include all ports */
};
static LIST_HEAD(dmar_atsr_units);
static LIST_HEAD(dmar_rmrr_units);
#define for_each_rmrr_units(rmrr) \
list_for_each_entry(rmrr, &dmar_rmrr_units, list)
static void flush_unmaps_timeout(unsigned long data);
static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
......@@ -389,6 +408,7 @@ struct deferred_flush_tables {
int next;
struct iova *iova[HIGH_WATER_MARK];
struct dmar_domain *domain[HIGH_WATER_MARK];
struct page *freelist[HIGH_WATER_MARK];
};
static struct deferred_flush_tables *deferred_flush;
......@@ -402,7 +422,12 @@ static LIST_HEAD(unmaps_to_do);
static int timer_on;
static long list_size;
static void domain_exit(struct dmar_domain *domain);
static void domain_remove_dev_info(struct dmar_domain *domain);
static void domain_remove_one_dev_info(struct dmar_domain *domain,
struct device *dev);
static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
struct device *dev);
#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
int dmar_disabled = 0;
......@@ -566,18 +591,31 @@ static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
static void domain_update_iommu_coherency(struct dmar_domain *domain)
{
int i;
i = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
int i, found = 0;
domain->iommu_coherency = i < g_num_of_iommus ? 1 : 0;
domain->iommu_coherency = 1;
for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
found = 1;
if (!ecap_coherent(g_iommus[i]->ecap)) {
domain->iommu_coherency = 0;
break;
}
}
if (found)
return;
/* No hardware attached; use lowest common denominator */
rcu_read_lock();
for_each_active_iommu(iommu, drhd) {
if (!ecap_coherent(iommu->ecap)) {
domain->iommu_coherency = 0;
break;
}
}
rcu_read_unlock();
}
static void domain_update_iommu_snooping(struct dmar_domain *domain)
......@@ -606,12 +644,15 @@ static void domain_update_iommu_superpage(struct dmar_domain *domain)
}
/* set iommu_superpage to the smallest common denominator */
rcu_read_lock();
for_each_active_iommu(iommu, drhd) {
mask &= cap_super_page_val(iommu->cap);
if (!mask) {
break;
}
}
rcu_read_unlock();
domain->iommu_superpage = fls(mask);
}
......@@ -623,32 +664,56 @@ static void domain_update_iommu_cap(struct dmar_domain *domain)
domain_update_iommu_superpage(domain);
}
static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
{
struct dmar_drhd_unit *drhd = NULL;
struct intel_iommu *iommu;
struct device *tmp;
struct pci_dev *ptmp, *pdev = NULL;
u16 segment;
int i;
for_each_active_drhd_unit(drhd) {
if (segment != drhd->segment)
if (dev_is_pci(dev)) {
pdev = to_pci_dev(dev);
segment = pci_domain_nr(pdev->bus);
} else if (ACPI_COMPANION(dev))
dev = &ACPI_COMPANION(dev)->dev;
rcu_read_lock();
for_each_active_iommu(iommu, drhd) {
if (pdev && segment != drhd->segment)
continue;
for (i = 0; i < drhd->devices_cnt; i++) {
if (drhd->devices[i] &&
drhd->devices[i]->bus->number == bus &&
drhd->devices[i]->devfn == devfn)
return drhd->iommu;
if (drhd->devices[i] &&
drhd->devices[i]->subordinate &&
drhd->devices[i]->subordinate->number <= bus &&
drhd->devices[i]->subordinate->busn_res.end >= bus)
return drhd->iommu;
for_each_active_dev_scope(drhd->devices,
drhd->devices_cnt, i, tmp) {
if (tmp == dev) {
*bus = drhd->devices[i].bus;
*devfn = drhd->devices[i].devfn;
goto out;
}
if (!pdev || !dev_is_pci(tmp))
continue;
ptmp = to_pci_dev(tmp);
if (ptmp->subordinate &&
ptmp->subordinate->number <= pdev->bus->number &&
ptmp->subordinate->busn_res.end >= pdev->bus->number)
goto got_pdev;
}
if (drhd->include_all)
return drhd->iommu;
if (pdev && drhd->include_all) {
got_pdev:
*bus = pdev->bus->number;
*devfn = pdev->devfn;
goto out;
}
}
iommu = NULL;
out:
rcu_read_unlock();
return NULL;
return iommu;
}
static void domain_flush_cache(struct dmar_domain *domain,
......@@ -748,7 +813,7 @@ static void free_context_table(struct intel_iommu *iommu)
}
static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
unsigned long pfn, int target_level)
unsigned long pfn, int *target_level)
{
int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
struct dma_pte *parent, *pte = NULL;
......@@ -763,14 +828,14 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
parent = domain->pgd;
while (level > 0) {
while (1) {
void *tmp_page;
offset = pfn_level_offset(pfn, level);
pte = &parent[offset];
if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
break;
if (level == target_level)
if (level == *target_level)
break;
if (!dma_pte_present(pte)) {
......@@ -791,10 +856,16 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
domain_flush_cache(domain, pte, sizeof(*pte));
}
}
if (level == 1)
break;
parent = phys_to_virt(dma_pte_addr(pte));
level--;
}
if (!*target_level)
*target_level = level;
return pte;
}
......@@ -832,7 +903,7 @@ static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
}
/* clear last level pte, a tlb flush should be followed */
static int dma_pte_clear_range(struct dmar_domain *domain,
static void dma_pte_clear_range(struct dmar_domain *domain,
unsigned long start_pfn,
unsigned long last_pfn)
{
......@@ -862,8 +933,6 @@ static int dma_pte_clear_range(struct dmar_domain *domain,
(void *)pte - (void *)first_pte);
} while (start_pfn && start_pfn <= last_pfn);
return min_t(int, (large_page - 1) * 9, MAX_AGAW_PFN_WIDTH);
}
static void dma_pte_free_level(struct dmar_domain *domain, int level,
......@@ -921,6 +990,123 @@ static void dma_pte_free_pagetable(struct dmar_domain *domain,
}
}
/* When a page at a given level is being unlinked from its parent, we don't
need to *modify* it at all. All we need to do is make a list of all the
pages which can be freed just as soon as we've flushed the IOTLB and we
know the hardware page-walk will no longer touch them.
The 'pte' argument is the *parent* PTE, pointing to the page that is to
be freed. */
static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
int level, struct dma_pte *pte,
struct page *freelist)
{
struct page *pg;
pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
pg->freelist = freelist;
freelist = pg;
if (level == 1)
return freelist;
for (pte = page_address(pg); !first_pte_in_page(pte); pte++) {
if (dma_pte_present(pte) && !dma_pte_superpage(pte))
freelist = dma_pte_list_pagetables(domain, level - 1,
pte, freelist);
}
return freelist;
}
static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
struct dma_pte *pte, unsigned long pfn,
unsigned long start_pfn,
unsigned long last_pfn,
struct page *freelist)
{
struct dma_pte *first_pte = NULL, *last_pte = NULL;
pfn = max(start_pfn, pfn);
pte = &pte[pfn_level_offset(pfn, level)];
do {
unsigned long level_pfn;
if (!dma_pte_present(pte))
goto next;
level_pfn = pfn & level_mask(level);
/* If range covers entire pagetable, free it */
if (start_pfn <= level_pfn &&
last_pfn >= level_pfn + level_size(level) - 1) {
/* These suborbinate page tables are going away entirely. Don't
bother to clear them; we're just going to *free* them. */
if (level > 1 && !dma_pte_superpage(pte))
freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
dma_clear_pte(pte);
if (!first_pte)
first_pte = pte;
last_pte = pte;
} else if (level > 1) {
/* Recurse down into a level that isn't *entirely* obsolete */
freelist = dma_pte_clear_level(domain, level - 1,
phys_to_virt(dma_pte_addr(pte)),
level_pfn, start_pfn, last_pfn,
freelist);
}
next:
pfn += level_size(level);
} while (!first_pte_in_page(++pte) && pfn <= last_pfn);
if (first_pte)
domain_flush_cache(domain, first_pte,
(void *)++last_pte - (void *)first_pte);
return freelist;
}
/* We can't just free the pages because the IOMMU may still be walking
the page tables, and may have cached the intermediate levels. The
pages can only be freed after the IOTLB flush has been done. */
struct page *domain_unmap(struct dmar_domain *domain,
unsigned long start_pfn,
unsigned long last_pfn)
{
int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
struct page *freelist = NULL;
BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
BUG_ON(start_pfn > last_pfn);
/* we don't need lock here; nobody else touches the iova range */
freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
domain->pgd, 0, start_pfn, last_pfn, NULL);
/* free pgd */
if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
struct page *pgd_page = virt_to_page(domain->pgd);
pgd_page->freelist = freelist;
freelist = pgd_page;
domain->pgd = NULL;
}
return freelist;
}
void dma_free_pagelist(struct page *freelist)
{
struct page *pg;
while ((pg = freelist)) {
freelist = pg->freelist;
free_pgtable_page(page_address(pg));
}
}
/* iommu handling */
static int iommu_alloc_root_entry(struct intel_iommu *iommu)
{
......@@ -1030,7 +1216,7 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
break;
case DMA_TLB_PSI_FLUSH:
val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
/* Note: always flush non-leaf currently */
/* IH bit is passed in as part of address */
val_iva = size_order | addr;
break;
default:
......@@ -1069,13 +1255,14 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
(unsigned long long)DMA_TLB_IAIG(val));
}
static struct device_domain_info *iommu_support_dev_iotlb(
struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
static struct device_domain_info *
iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
u8 bus, u8 devfn)
{
int found = 0;
unsigned long flags;
struct device_domain_info *info;
struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
struct pci_dev *pdev;
if (!ecap_dev_iotlb_support(iommu->ecap))
return NULL;
......@@ -1091,34 +1278,35 @@ static struct device_domain_info *iommu_support_dev_iotlb(
}
spin_unlock_irqrestore(&device_domain_lock, flags);
if (!found || !info->dev)
if (!found || !info->dev || !dev_is_pci(info->dev))
return NULL;
if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
return NULL;
pdev = to_pci_dev(info->dev);
if (!dmar_find_matched_atsr_unit(info->dev))
if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
return NULL;
info->iommu = iommu;
if (!dmar_find_matched_atsr_unit(pdev))
return NULL;
return info;
}
static void iommu_enable_dev_iotlb(struct device_domain_info *info)
{
if (!info)
if (!info || !dev_is_pci(info->dev))
return;
pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
}
static void iommu_disable_dev_iotlb(struct device_domain_info *info)
{
if (!info->dev || !pci_ats_enabled(info->dev))
if (!info->dev || !dev_is_pci(info->dev) ||
!pci_ats_enabled(to_pci_dev(info->dev)))
return;
pci_disable_ats(info->dev);
pci_disable_ats(to_pci_dev(info->dev));
}
static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
......@@ -1130,24 +1318,31 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
spin_lock_irqsave(&device_domain_lock, flags);
list_for_each_entry(info, &domain->devices, link) {
if (!info->dev || !pci_ats_enabled(info->dev))
struct pci_dev *pdev;
if (!info->dev || !dev_is_pci(info->dev))
continue;
pdev = to_pci_dev(info->dev);
if (!pci_ats_enabled(pdev))
continue;
sid = info->bus << 8 | info->devfn;
qdep = pci_ats_queue_depth(info->dev);
qdep = pci_ats_queue_depth(pdev);
qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
}
spin_unlock_irqrestore(&device_domain_lock, flags);
}
static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
unsigned long pfn, unsigned int pages, int map)
unsigned long pfn, unsigned int pages, int ih, int map)
{
unsigned int mask = ilog2(__roundup_pow_of_two(pages));
uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
BUG_ON(pages == 0);
if (ih)
ih = 1 << 6;
/*
* Fallback to domain selective flush if no PSI support or the size is
* too big.
......@@ -1158,7 +1353,7 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
iommu->flush.flush_iotlb(iommu, did, 0, 0,
DMA_TLB_DSI_FLUSH);
else
iommu->flush.flush_iotlb(iommu, did, addr, mask,
iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
DMA_TLB_PSI_FLUSH);
/*
......@@ -1261,10 +1456,6 @@ static int iommu_init_domains(struct intel_iommu *iommu)
return 0;
}
static void domain_exit(struct dmar_domain *domain);
static void vm_domain_exit(struct dmar_domain *domain);
static void free_dmar_iommu(struct intel_iommu *iommu)
{
struct dmar_domain *domain;
......@@ -1273,18 +1464,21 @@ static void free_dmar_iommu(struct intel_iommu *iommu)
if ((iommu->domains) && (iommu->domain_ids)) {
for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
/*
* Domain id 0 is reserved for invalid translation
* if hardware supports caching mode.
*/
if (cap_caching_mode(iommu->cap) && i == 0)
continue;
domain = iommu->domains[i];
clear_bit(i, iommu->domain_ids);
spin_lock_irqsave(&domain->iommu_lock, flags);
count = --domain->iommu_count;
spin_unlock_irqrestore(&domain->iommu_lock, flags);
if (count == 0) {
if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
vm_domain_exit(domain);
else
domain_exit(domain);
}
if (count == 0)
domain_exit(domain);
}
}
......@@ -1298,21 +1492,14 @@ static void free_dmar_iommu(struct intel_iommu *iommu)
g_iommus[iommu->seq_id] = NULL;
/* if all iommus are freed, free g_iommus */
for (i = 0; i < g_num_of_iommus; i++) {
if (g_iommus[i])
break;
}
if (i == g_num_of_iommus)
kfree(g_iommus);
/* free context mapping */
free_context_table(iommu);
}
static struct dmar_domain *alloc_domain(void)
static struct dmar_domain *alloc_domain(bool vm)
{
/* domain id for virtual machine, it won't be set in context */
static atomic_t vm_domid = ATOMIC_INIT(0);
struct dmar_domain *domain;
domain = alloc_domain_mem();
......@@ -1320,8 +1507,15 @@ static struct dmar_domain *alloc_domain(void)
return NULL;
domain->nid = -1;
domain->iommu_count = 0;
memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
domain->flags = 0;
spin_lock_init(&domain->iommu_lock);
INIT_LIST_HEAD(&domain->devices);
if (vm) {
domain->id = atomic_inc_return(&vm_domid);
domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
}
return domain;
}
......@@ -1345,6 +1539,7 @@ static int iommu_attach_domain(struct dmar_domain *domain,
}
domain->id = num;
domain->iommu_count++;
set_bit(num, iommu->domain_ids);
set_bit(iommu->seq_id, domain->iommu_bmp);
iommu->domains[num] = domain;
......@@ -1358,22 +1553,16 @@ static void iommu_detach_domain(struct dmar_domain *domain,
{
unsigned long flags;
int num, ndomains;
int found = 0;
spin_lock_irqsave(&iommu->lock, flags);
ndomains = cap_ndoms(iommu->cap);
for_each_set_bit(num, iommu->domain_ids, ndomains) {
if (iommu->domains[num] == domain) {
found = 1;
clear_bit(num, iommu->domain_ids);
iommu->domains[num] = NULL;
break;
}
}
if (found) {
clear_bit(num, iommu->domain_ids);
clear_bit(iommu->seq_id, domain->iommu_bmp);
iommu->domains[num] = NULL;
}
spin_unlock_irqrestore(&iommu->lock, flags);
}
......@@ -1445,8 +1634,6 @@ static int domain_init(struct dmar_domain *domain, int guest_width)
unsigned long sagaw;
init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
spin_lock_init(&domain->iommu_lock);
domain_reserve_special_ranges(domain);
/* calculate AGAW */
......@@ -1465,7 +1652,6 @@ static int domain_init(struct dmar_domain *domain, int guest_width)
return -ENODEV;
}
domain->agaw = agaw;
INIT_LIST_HEAD(&domain->devices);
if (ecap_coherent(iommu->ecap))
domain->iommu_coherency = 1;
......@@ -1477,8 +1663,11 @@ static int domain_init(struct dmar_domain *domain, int guest_width)
else
domain->iommu_snooping = 0;
domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
domain->iommu_count = 1;
if (intel_iommu_superpage)
domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
else
domain->iommu_superpage = 0;
domain->nid = iommu->node;
/* always allocate the top pgd */
......@@ -1493,6 +1682,7 @@ static void domain_exit(struct dmar_domain *domain)
{
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
struct page *freelist = NULL;
/* Domain 0 is reserved, so dont process it */
if (!domain)
......@@ -1502,29 +1692,33 @@ static void domain_exit(struct dmar_domain *domain)
if (!intel_iommu_strict)
flush_unmaps_timeout(0);
/* remove associated devices */
domain_remove_dev_info(domain);
/* destroy iovas */
put_iova_domain(&domain->iovad);
/* clear ptes */
dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
/* free page tables */
dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
/* clear attached or cached domains */
rcu_read_lock();
for_each_active_iommu(iommu, drhd)
if (test_bit(iommu->seq_id, domain->iommu_bmp))
if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
test_bit(iommu->seq_id, domain->iommu_bmp))
iommu_detach_domain(domain, iommu);
rcu_read_unlock();
dma_free_pagelist(freelist);
free_domain_mem(domain);
}
static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
u8 bus, u8 devfn, int translation)
static int domain_context_mapping_one(struct dmar_domain *domain,
struct intel_iommu *iommu,
u8 bus, u8 devfn, int translation)
{
struct context_entry *context;
unsigned long flags;
struct intel_iommu *iommu;
struct dma_pte *pgd;
unsigned long num;
unsigned long ndomains;
......@@ -1539,10 +1733,6 @@ static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
translation != CONTEXT_TT_MULTI_LEVEL);
iommu = device_to_iommu(segment, bus, devfn);
if (!iommu)
return -ENODEV;
context = device_to_context_entry(iommu, bus, devfn);
if (!context)
return -ENOMEM;
......@@ -1600,7 +1790,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
context_set_domain_id(context, id);
if (translation != CONTEXT_TT_PASS_THROUGH) {
info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
translation = info ? CONTEXT_TT_DEV_IOTLB :
CONTEXT_TT_MULTI_LEVEL;
}
......@@ -1650,27 +1840,32 @@ static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
}
static int
domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
int translation)
domain_context_mapping(struct dmar_domain *domain, struct device *dev,
int translation)
{
int ret;
struct pci_dev *tmp, *parent;
struct pci_dev *pdev, *tmp, *parent;
struct intel_iommu *iommu;
u8 bus, devfn;
iommu = device_to_iommu(dev, &bus, &devfn);
if (!iommu)
return -ENODEV;
ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
pdev->bus->number, pdev->devfn,
ret = domain_context_mapping_one(domain, iommu, bus, devfn,
translation);
if (ret)
if (ret || !dev_is_pci(dev))
return ret;
/* dependent device mapping */
pdev = to_pci_dev(dev);
tmp = pci_find_upstream_pcie_bridge(pdev);
if (!tmp)
return 0;
/* Secondary interface's bus number and devfn 0 */
parent = pdev->bus->self;
while (parent != tmp) {
ret = domain_context_mapping_one(domain,
pci_domain_nr(parent->bus),
ret = domain_context_mapping_one(domain, iommu,
parent->bus->number,
parent->devfn, translation);
if (ret)
......@@ -1678,33 +1873,33 @@ domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
parent = parent->bus->self;
}
if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
return domain_context_mapping_one(domain,
pci_domain_nr(tmp->subordinate),
return domain_context_mapping_one(domain, iommu,
tmp->subordinate->number, 0,
translation);
else /* this is a legacy PCI bridge */
return domain_context_mapping_one(domain,
pci_domain_nr(tmp->bus),
return domain_context_mapping_one(domain, iommu,
tmp->bus->number,
tmp->devfn,
translation);
}
static int domain_context_mapped(struct pci_dev *pdev)
static int domain_context_mapped(struct device *dev)
{
int ret;
struct pci_dev *tmp, *parent;
struct pci_dev *pdev, *tmp, *parent;
struct intel_iommu *iommu;
u8 bus, devfn;
iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
pdev->devfn);
iommu = device_to_iommu(dev, &bus, &devfn);
if (!iommu)
return -ENODEV;
ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
if (!ret)
ret = device_context_mapped(iommu, bus, devfn);
if (!ret || !dev_is_pci(dev))
return ret;
/* dependent device mapping */
pdev = to_pci_dev(dev);
tmp = pci_find_upstream_pcie_bridge(pdev);
if (!tmp)
return ret;
......@@ -1800,7 +1995,7 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
if (!pte) {
largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
if (!pte)
return -ENOMEM;
/* It is large page*/
......@@ -1899,14 +2094,13 @@ static inline void unlink_domain_info(struct device_domain_info *info)
list_del(&info->link);
list_del(&info->global);
if (info->dev)
info->dev->dev.archdata.iommu = NULL;
info->dev->archdata.iommu = NULL;
}
static void domain_remove_dev_info(struct dmar_domain *domain)
{
struct device_domain_info *info;
unsigned long flags;
struct intel_iommu *iommu;
unsigned long flags, flags2;
spin_lock_irqsave(&device_domain_lock, flags);
while (!list_empty(&domain->devices)) {
......@@ -1916,10 +2110,23 @@ static void domain_remove_dev_info(struct dmar_domain *domain)
spin_unlock_irqrestore(&device_domain_lock, flags);
iommu_disable_dev_iotlb(info);
iommu = device_to_iommu(info->segment, info->bus, info->devfn);
iommu_detach_dev(iommu, info->bus, info->devfn);
free_devinfo_mem(info);
iommu_detach_dev(info->iommu, info->bus, info->devfn);
if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
iommu_detach_dependent_devices(info->iommu, info->dev);
/* clear this iommu in iommu_bmp, update iommu count
* and capabilities
*/
spin_lock_irqsave(&domain->iommu_lock, flags2);
if (test_and_clear_bit(info->iommu->seq_id,
domain->iommu_bmp)) {
domain->iommu_count--;
domain_update_iommu_cap(domain);
}
spin_unlock_irqrestore(&domain->iommu_lock, flags2);
}
free_devinfo_mem(info);
spin_lock_irqsave(&device_domain_lock, flags);
}
spin_unlock_irqrestore(&device_domain_lock, flags);
......@@ -1927,155 +2134,151 @@ static void domain_remove_dev_info(struct dmar_domain *domain)
/*
* find_domain
* Note: we use struct pci_dev->dev.archdata.iommu stores the info
* Note: we use struct device->archdata.iommu stores the info
*/
static struct dmar_domain *
find_domain(struct pci_dev *pdev)
static struct dmar_domain *find_domain(struct device *dev)
{
struct device_domain_info *info;
/* No lock here, assumes no domain exit in normal case */
info = pdev->dev.archdata.iommu;
info = dev->archdata.iommu;
if (info)
return info->domain;
return NULL;
}
static inline struct device_domain_info *
dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
{
struct device_domain_info *info;
list_for_each_entry(info, &device_domain_list, global)
if (info->iommu->segment == segment && info->bus == bus &&
info->devfn == devfn)
return info;
return NULL;
}
static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
int bus, int devfn,
struct device *dev,
struct dmar_domain *domain)
{
struct dmar_domain *found = NULL;
struct device_domain_info *info;
unsigned long flags;
info = alloc_devinfo_mem();
if (!info)
return NULL;
info->bus = bus;
info->devfn = devfn;
info->dev = dev;
info->domain = domain;
info->iommu = iommu;
if (!dev)
domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
spin_lock_irqsave(&device_domain_lock, flags);
if (dev)
found = find_domain(dev);
else {
struct device_domain_info *info2;
info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
if (info2)
found = info2->domain;
}
if (found) {
spin_unlock_irqrestore(&device_domain_lock, flags);
free_devinfo_mem(info);
/* Caller must free the original domain */
return found;
}
list_add(&info->link, &domain->devices);
list_add(&info->global, &device_domain_list);
if (dev)
dev->archdata.iommu = info;
spin_unlock_irqrestore(&device_domain_lock, flags);
return domain;
}
/* domain is initialized */
static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
{
struct dmar_domain *domain, *found = NULL;
struct intel_iommu *iommu;
struct dmar_drhd_unit *drhd;
struct device_domain_info *info, *tmp;
struct pci_dev *dev_tmp;
struct dmar_domain *domain, *free = NULL;
struct intel_iommu *iommu = NULL;
struct device_domain_info *info;
struct pci_dev *dev_tmp = NULL;
unsigned long flags;
int bus = 0, devfn = 0;
int segment;
int ret;
u8 bus, devfn, bridge_bus, bridge_devfn;
domain = find_domain(pdev);
domain = find_domain(dev);
if (domain)
return domain;
segment = pci_domain_nr(pdev->bus);
if (dev_is_pci(dev)) {
struct pci_dev *pdev = to_pci_dev(dev);
u16 segment;
dev_tmp = pci_find_upstream_pcie_bridge(pdev);
if (dev_tmp) {
if (pci_is_pcie(dev_tmp)) {
bus = dev_tmp->subordinate->number;
devfn = 0;
} else {
bus = dev_tmp->bus->number;
devfn = dev_tmp->devfn;
}
spin_lock_irqsave(&device_domain_lock, flags);
list_for_each_entry(info, &device_domain_list, global) {
if (info->segment == segment &&
info->bus == bus && info->devfn == devfn) {
found = info->domain;
break;
segment = pci_domain_nr(pdev->bus);
dev_tmp = pci_find_upstream_pcie_bridge(pdev);
if (dev_tmp) {
if (pci_is_pcie(dev_tmp)) {
bridge_bus = dev_tmp->subordinate->number;
bridge_devfn = 0;
} else {
bridge_bus = dev_tmp->bus->number;
bridge_devfn = dev_tmp->devfn;
}
}
spin_unlock_irqrestore(&device_domain_lock, flags);
/* pcie-pci bridge already has a domain, uses it */
if (found) {
domain = found;
goto found_domain;
spin_lock_irqsave(&device_domain_lock, flags);
info = dmar_search_domain_by_dev_info(segment, bus, devfn);
if (info) {
iommu = info->iommu;
domain = info->domain;
}
spin_unlock_irqrestore(&device_domain_lock, flags);
/* pcie-pci bridge already has a domain, uses it */
if (info)
goto found_domain;
}
}
domain = alloc_domain();
if (!domain)
iommu = device_to_iommu(dev, &bus, &devfn);
if (!iommu)
goto error;
/* Allocate new domain for the device */
drhd = dmar_find_matched_drhd_unit(pdev);
if (!drhd) {
printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
pci_name(pdev));
free_domain_mem(domain);
return NULL;
}
iommu = drhd->iommu;
ret = iommu_attach_domain(domain, iommu);
if (ret) {
/* Allocate and initialize new domain for the device */
domain = alloc_domain(false);
if (!domain)
goto error;
if (iommu_attach_domain(domain, iommu)) {
free_domain_mem(domain);
domain = NULL;
goto error;
}
if (domain_init(domain, gaw)) {
domain_exit(domain);
free = domain;
if (domain_init(domain, gaw))
goto error;
}
/* register pcie-to-pci device */
if (dev_tmp) {
info = alloc_devinfo_mem();
if (!info) {
domain_exit(domain);
domain = dmar_insert_dev_info(iommu, bridge_bus, bridge_devfn,
NULL, domain);
if (!domain)
goto error;
}
info->segment = segment;
info->bus = bus;
info->devfn = devfn;
info->dev = NULL;
info->domain = domain;
/* This domain is shared by devices under p2p bridge */
domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
/* pcie-to-pci bridge already has a domain, uses it */
found = NULL;
spin_lock_irqsave(&device_domain_lock, flags);
list_for_each_entry(tmp, &device_domain_list, global) {
if (tmp->segment == segment &&
tmp->bus == bus && tmp->devfn == devfn) {
found = tmp->domain;
break;
}
}
if (found) {
spin_unlock_irqrestore(&device_domain_lock, flags);
free_devinfo_mem(info);
domain_exit(domain);
domain = found;
} else {
list_add(&info->link, &domain->devices);
list_add(&info->global, &device_domain_list);
spin_unlock_irqrestore(&device_domain_lock, flags);
}
}
found_domain:
info = alloc_devinfo_mem();
if (!info)
goto error;
info->segment = segment;
info->bus = pdev->bus->number;
info->devfn = pdev->devfn;
info->dev = pdev;
info->domain = domain;
spin_lock_irqsave(&device_domain_lock, flags);
/* somebody is fast */
found = find_domain(pdev);
if (found != NULL) {
spin_unlock_irqrestore(&device_domain_lock, flags);
if (found != domain) {
domain_exit(domain);
domain = found;
}
free_devinfo_mem(info);
return domain;
}
list_add(&info->link, &domain->devices);
list_add(&info->global, &device_domain_list);
pdev->dev.archdata.iommu = info;
spin_unlock_irqrestore(&device_domain_lock, flags);
return domain;
domain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
error:
/* recheck it here, maybe others set it */
return find_domain(pdev);
if (free != domain)
domain_exit(free);
return domain;
}
static int iommu_identity_mapping;
......@@ -2109,14 +2312,14 @@ static int iommu_domain_identity_map(struct dmar_domain *domain,
DMA_PTE_READ|DMA_PTE_WRITE);
}
static int iommu_prepare_identity_map(struct pci_dev *pdev,
static int iommu_prepare_identity_map(struct device *dev,
unsigned long long start,
unsigned long long end)
{
struct dmar_domain *domain;
int ret;
domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
if (!domain)
return -ENOMEM;
......@@ -2126,13 +2329,13 @@ static int iommu_prepare_identity_map(struct pci_dev *pdev,
up to start with in si_domain */
if (domain == si_domain && hw_pass_through) {
printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
pci_name(pdev), start, end);
dev_name(dev), start, end);
return 0;
}
printk(KERN_INFO
"IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
pci_name(pdev), start, end);
dev_name(dev), start, end);
if (end < start) {
WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
......@@ -2160,7 +2363,7 @@ static int iommu_prepare_identity_map(struct pci_dev *pdev,
goto error;
/* context entry init */
ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
if (ret)
goto error;
......@@ -2172,12 +2375,12 @@ static int iommu_prepare_identity_map(struct pci_dev *pdev,
}
static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
struct pci_dev *pdev)
struct device *dev)
{
if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
return 0;
return iommu_prepare_identity_map(pdev, rmrr->base_address,
rmrr->end_address);
return iommu_prepare_identity_map(dev, rmrr->base_address,
rmrr->end_address);
}
#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
......@@ -2191,7 +2394,7 @@ static inline void iommu_prepare_isa(void)
return;
printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
if (ret)
printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
......@@ -2213,10 +2416,12 @@ static int __init si_domain_init(int hw)
struct intel_iommu *iommu;
int nid, ret = 0;
si_domain = alloc_domain();
si_domain = alloc_domain(false);
if (!si_domain)
return -EFAULT;
si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
for_each_active_iommu(iommu, drhd) {
ret = iommu_attach_domain(si_domain, iommu);
if (ret) {
......@@ -2230,7 +2435,6 @@ static int __init si_domain_init(int hw)
return -EFAULT;
}
si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
pr_debug("IOMMU: identity mapping domain is domain %d\n",
si_domain->id);
......@@ -2252,16 +2456,14 @@ static int __init si_domain_init(int hw)
return 0;
}
static void domain_remove_one_dev_info(struct dmar_domain *domain,
struct pci_dev *pdev);
static int identity_mapping(struct pci_dev *pdev)
static int identity_mapping(struct device *dev)
{
struct device_domain_info *info;
if (likely(!iommu_identity_mapping))
return 0;
info = pdev->dev.archdata.iommu;
info = dev->archdata.iommu;
if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
return (info->domain == si_domain);
......@@ -2269,111 +2471,112 @@ static int identity_mapping(struct pci_dev *pdev)
}
static int domain_add_dev_info(struct dmar_domain *domain,
struct pci_dev *pdev,
int translation)
struct device *dev, int translation)
{
struct device_domain_info *info;
unsigned long flags;
struct dmar_domain *ndomain;
struct intel_iommu *iommu;
u8 bus, devfn;
int ret;
info = alloc_devinfo_mem();
if (!info)
return -ENOMEM;
iommu = device_to_iommu(dev, &bus, &devfn);
if (!iommu)
return -ENODEV;
info->segment = pci_domain_nr(pdev->bus);
info->bus = pdev->bus->number;
info->devfn = pdev->devfn;
info->dev = pdev;
info->domain = domain;
ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
if (ndomain != domain)
return -EBUSY;
spin_lock_irqsave(&device_domain_lock, flags);
list_add(&info->link, &domain->devices);
list_add(&info->global, &device_domain_list);
pdev->dev.archdata.iommu = info;
spin_unlock_irqrestore(&device_domain_lock, flags);
ret = domain_context_mapping(domain, pdev, translation);
ret = domain_context_mapping(domain, dev, translation);
if (ret) {
spin_lock_irqsave(&device_domain_lock, flags);
unlink_domain_info(info);
spin_unlock_irqrestore(&device_domain_lock, flags);
free_devinfo_mem(info);
domain_remove_one_dev_info(domain, dev);
return ret;
}
return 0;
}
static bool device_has_rmrr(struct pci_dev *dev)
static bool device_has_rmrr(struct device *dev)
{
struct dmar_rmrr_unit *rmrr;
struct device *tmp;
int i;
rcu_read_lock();
for_each_rmrr_units(rmrr) {
for (i = 0; i < rmrr->devices_cnt; i++) {
/*
* Return TRUE if this RMRR contains the device that
* is passed in.
*/
if (rmrr->devices[i] == dev)
/*
* Return TRUE if this RMRR contains the device that
* is passed in.
*/
for_each_active_dev_scope(rmrr->devices,
rmrr->devices_cnt, i, tmp)
if (tmp == dev) {
rcu_read_unlock();
return true;
}
}
}
rcu_read_unlock();
return false;
}
static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
static int iommu_should_identity_map(struct device *dev, int startup)
{
/*
* We want to prevent any device associated with an RMRR from
* getting placed into the SI Domain. This is done because
* problems exist when devices are moved in and out of domains
* and their respective RMRR info is lost. We exempt USB devices
* from this process due to their usage of RMRRs that are known
* to not be needed after BIOS hand-off to OS.
*/
if (device_has_rmrr(pdev) &&
(pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
return 0;
if (dev_is_pci(dev)) {
struct pci_dev *pdev = to_pci_dev(dev);
if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
return 1;
/*
* We want to prevent any device associated with an RMRR from
* getting placed into the SI Domain. This is done because
* problems exist when devices are moved in and out of domains
* and their respective RMRR info is lost. We exempt USB devices
* from this process due to their usage of RMRRs that are known
* to not be needed after BIOS hand-off to OS.
*/
if (device_has_rmrr(dev) &&
(pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
return 0;
if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
return 1;
if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
return 1;
if (!(iommu_identity_mapping & IDENTMAP_ALL))
return 0;
if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
return 1;
/*
* We want to start off with all devices in the 1:1 domain, and
* take them out later if we find they can't access all of memory.
*
* However, we can't do this for PCI devices behind bridges,
* because all PCI devices behind the same bridge will end up
* with the same source-id on their transactions.
*
* Practically speaking, we can't change things around for these
* devices at run-time, because we can't be sure there'll be no
* DMA transactions in flight for any of their siblings.
*
* So PCI devices (unless they're on the root bus) as well as
* their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
* the 1:1 domain, just in _case_ one of their siblings turns out
* not to be able to map all of memory.
*/
if (!pci_is_pcie(pdev)) {
if (!pci_is_root_bus(pdev->bus))
if (!(iommu_identity_mapping & IDENTMAP_ALL))
return 0;
if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
/*
* We want to start off with all devices in the 1:1 domain, and
* take them out later if we find they can't access all of memory.
*
* However, we can't do this for PCI devices behind bridges,
* because all PCI devices behind the same bridge will end up
* with the same source-id on their transactions.
*
* Practically speaking, we can't change things around for these
* devices at run-time, because we can't be sure there'll be no
* DMA transactions in flight for any of their siblings.
*
* So PCI devices (unless they're on the root bus) as well as
* their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
* the 1:1 domain, just in _case_ one of their siblings turns out
* not to be able to map all of memory.
*/
if (!pci_is_pcie(pdev)) {
if (!pci_is_root_bus(pdev->bus))
return 0;
if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
return 0;
} else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
return 0;
} else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
return 0;
} else {
if (device_has_rmrr(dev))
return 0;
}
/*
/*
* At boot time, we don't yet know if devices will be 64-bit capable.
* Assume that they will -- if they turn out not to be, then we can
* Assume that they will — if they turn out not to be, then we can
* take them out of the 1:1 domain later.
*/
if (!startup) {
......@@ -2381,42 +2584,77 @@ static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
* If the device's dma_mask is less than the system's memory
* size then this is not a candidate for identity mapping.
*/
u64 dma_mask = pdev->dma_mask;
u64 dma_mask = *dev->dma_mask;
if (pdev->dev.coherent_dma_mask &&
pdev->dev.coherent_dma_mask < dma_mask)
dma_mask = pdev->dev.coherent_dma_mask;
if (dev->coherent_dma_mask &&
dev->coherent_dma_mask < dma_mask)
dma_mask = dev->coherent_dma_mask;
return dma_mask >= dma_get_required_mask(&pdev->dev);
return dma_mask >= dma_get_required_mask(dev);
}
return 1;
}
static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
{
int ret;
if (!iommu_should_identity_map(dev, 1))
return 0;
ret = domain_add_dev_info(si_domain, dev,
hw ? CONTEXT_TT_PASS_THROUGH :
CONTEXT_TT_MULTI_LEVEL);
if (!ret)
pr_info("IOMMU: %s identity mapping for device %s\n",
hw ? "hardware" : "software", dev_name(dev));
else if (ret == -ENODEV)
/* device not associated with an iommu */
ret = 0;
return ret;
}
static int __init iommu_prepare_static_identity_mapping(int hw)
{
struct pci_dev *pdev = NULL;
int ret;
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
struct device *dev;
int i;
int ret = 0;
ret = si_domain_init(hw);
if (ret)
return -EFAULT;
for_each_pci_dev(pdev) {
if (iommu_should_identity_map(pdev, 1)) {
ret = domain_add_dev_info(si_domain, pdev,
hw ? CONTEXT_TT_PASS_THROUGH :
CONTEXT_TT_MULTI_LEVEL);
if (ret) {
/* device not associated with an iommu */
if (ret == -ENODEV)
continue;
return ret;
ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
if (ret)
return ret;
}
for_each_active_iommu(iommu, drhd)
for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
struct acpi_device_physical_node *pn;
struct acpi_device *adev;
if (dev->bus != &acpi_bus_type)
continue;
adev= to_acpi_device(dev);
mutex_lock(&adev->physical_node_lock);
list_for_each_entry(pn, &adev->physical_node_list, node) {
ret = dev_prepare_static_identity_mapping(pn->dev, hw);
if (ret)
break;
}
pr_info("IOMMU: %s identity mapping for device %s\n",
hw ? "hardware" : "software", pci_name(pdev));
mutex_unlock(&adev->physical_node_lock);
if (ret)
return ret;
}
}
return 0;
}
......@@ -2425,7 +2663,7 @@ static int __init init_dmars(void)
{
struct dmar_drhd_unit *drhd;
struct dmar_rmrr_unit *rmrr;
struct pci_dev *pdev;
struct device *dev;
struct intel_iommu *iommu;
int i, ret;
......@@ -2461,7 +2699,7 @@ static int __init init_dmars(void)
sizeof(struct deferred_flush_tables), GFP_KERNEL);
if (!deferred_flush) {
ret = -ENOMEM;
goto error;
goto free_g_iommus;
}
for_each_active_iommu(iommu, drhd) {
......@@ -2469,7 +2707,7 @@ static int __init init_dmars(void)
ret = iommu_init_domains(iommu);
if (ret)
goto error;
goto free_iommu;
/*
* TBD:
......@@ -2479,7 +2717,7 @@ static int __init init_dmars(void)
ret = iommu_alloc_root_entry(iommu);
if (ret) {
printk(KERN_ERR "IOMMU: allocate root entry failed\n");
goto error;
goto free_iommu;
}
if (!ecap_pass_through(iommu->ecap))
hw_pass_through = 0;
......@@ -2548,7 +2786,7 @@ static int __init init_dmars(void)
ret = iommu_prepare_static_identity_mapping(hw_pass_through);
if (ret) {
printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
goto error;
goto free_iommu;
}
}
/*
......@@ -2567,15 +2805,10 @@ static int __init init_dmars(void)
*/
printk(KERN_INFO "IOMMU: Setting RMRR:\n");
for_each_rmrr_units(rmrr) {
for (i = 0; i < rmrr->devices_cnt; i++) {
pdev = rmrr->devices[i];
/*
* some BIOS lists non-exist devices in DMAR
* table.
*/
if (!pdev)
continue;
ret = iommu_prepare_rmrr_dev(rmrr, pdev);
/* some BIOS lists non-exist devices in DMAR table. */
for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
i, dev) {
ret = iommu_prepare_rmrr_dev(rmrr, dev);
if (ret)
printk(KERN_ERR
"IOMMU: mapping reserved region failed\n");
......@@ -2606,7 +2839,7 @@ static int __init init_dmars(void)
ret = dmar_set_interrupt(iommu);
if (ret)
goto error;
goto free_iommu;
iommu_set_root_entry(iommu);
......@@ -2615,17 +2848,20 @@ static int __init init_dmars(void)
ret = iommu_enable_translation(iommu);
if (ret)
goto error;
goto free_iommu;
iommu_disable_protect_mem_regions(iommu);
}
return 0;
error:
free_iommu:
for_each_active_iommu(iommu, drhd)
free_dmar_iommu(iommu);
kfree(deferred_flush);
free_g_iommus:
kfree(g_iommus);
error:
return ret;
}
......@@ -2634,7 +2870,6 @@ static struct iova *intel_alloc_iova(struct device *dev,
struct dmar_domain *domain,
unsigned long nrpages, uint64_t dma_mask)
{
struct pci_dev *pdev = to_pci_dev(dev);
struct iova *iova = NULL;
/* Restrict dma_mask to the width that the iommu can handle */
......@@ -2654,34 +2889,31 @@ static struct iova *intel_alloc_iova(struct device *dev,
iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
if (unlikely(!iova)) {
printk(KERN_ERR "Allocating %ld-page iova for %s failed",
nrpages, pci_name(pdev));
nrpages, dev_name(dev));
return NULL;
}
return iova;
}
static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
{
struct dmar_domain *domain;
int ret;
domain = get_domain_for_dev(pdev,
DEFAULT_DOMAIN_ADDRESS_WIDTH);
domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
if (!domain) {
printk(KERN_ERR
"Allocating domain for %s failed", pci_name(pdev));
printk(KERN_ERR "Allocating domain for %s failed",
dev_name(dev));
return NULL;
}
/* make sure context mapping is ok */
if (unlikely(!domain_context_mapped(pdev))) {
ret = domain_context_mapping(domain, pdev,
CONTEXT_TT_MULTI_LEVEL);
if (unlikely(!domain_context_mapped(dev))) {
ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
if (ret) {
printk(KERN_ERR
"Domain context map for %s failed",
pci_name(pdev));
printk(KERN_ERR "Domain context map for %s failed",
dev_name(dev));
return NULL;
}
}
......@@ -2689,51 +2921,46 @@ static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
return domain;
}
static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
{
struct device_domain_info *info;
/* No lock here, assumes no domain exit in normal case */
info = dev->dev.archdata.iommu;
info = dev->archdata.iommu;
if (likely(info))
return info->domain;
return __get_valid_domain_for_dev(dev);
}
static int iommu_dummy(struct pci_dev *pdev)
static int iommu_dummy(struct device *dev)
{
return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
}
/* Check if the pdev needs to go through non-identity map and unmap process.*/
/* Check if the dev needs to go through non-identity map and unmap process.*/
static int iommu_no_mapping(struct device *dev)
{
struct pci_dev *pdev;
int found;
if (unlikely(!dev_is_pci(dev)))
return 1;
pdev = to_pci_dev(dev);
if (iommu_dummy(pdev))
if (iommu_dummy(dev))
return 1;
if (!iommu_identity_mapping)
return 0;
found = identity_mapping(pdev);
found = identity_mapping(dev);
if (found) {
if (iommu_should_identity_map(pdev, 0))
if (iommu_should_identity_map(dev, 0))
return 1;
else {
/*
* 32 bit DMA is removed from si_domain and fall back
* to non-identity mapping.
*/
domain_remove_one_dev_info(si_domain, pdev);
domain_remove_one_dev_info(si_domain, dev);
printk(KERN_INFO "32bit %s uses non-identity mapping\n",
pci_name(pdev));
dev_name(dev));
return 0;
}
} else {
......@@ -2741,15 +2968,15 @@ static int iommu_no_mapping(struct device *dev)
* In case of a detached 64 bit DMA device from vm, the device
* is put into si_domain for identity mapping.
*/
if (iommu_should_identity_map(pdev, 0)) {
if (iommu_should_identity_map(dev, 0)) {
int ret;
ret = domain_add_dev_info(si_domain, pdev,
ret = domain_add_dev_info(si_domain, dev,
hw_pass_through ?
CONTEXT_TT_PASS_THROUGH :
CONTEXT_TT_MULTI_LEVEL);
if (!ret) {
printk(KERN_INFO "64bit %s uses identity mapping\n",
pci_name(pdev));
dev_name(dev));
return 1;
}
}
......@@ -2758,10 +2985,9 @@ static int iommu_no_mapping(struct device *dev)
return 0;
}
static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
size_t size, int dir, u64 dma_mask)
{
struct pci_dev *pdev = to_pci_dev(hwdev);
struct dmar_domain *domain;
phys_addr_t start_paddr;
struct iova *iova;
......@@ -2772,17 +2998,17 @@ static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
BUG_ON(dir == DMA_NONE);
if (iommu_no_mapping(hwdev))
if (iommu_no_mapping(dev))
return paddr;
domain = get_valid_domain_for_dev(pdev);
domain = get_valid_domain_for_dev(dev);
if (!domain)
return 0;
iommu = domain_get_iommu(domain);
size = aligned_nrpages(paddr, size);
iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
if (!iova)
goto error;
......@@ -2808,7 +3034,7 @@ static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
/* it's a non-present to present mapping. Only flush if caching mode */
if (cap_caching_mode(iommu->cap))
iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
else
iommu_flush_write_buffer(iommu);
......@@ -2820,7 +3046,7 @@ static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
if (iova)
__free_iova(&domain->iovad, iova);
printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
pci_name(pdev), size, (unsigned long long)paddr, dir);
dev_name(dev), size, (unsigned long long)paddr, dir);
return 0;
}
......@@ -2830,7 +3056,7 @@ static dma_addr_t intel_map_page(struct device *dev, struct page *page,
struct dma_attrs *attrs)
{
return __intel_map_single(dev, page_to_phys(page) + offset, size,
dir, to_pci_dev(dev)->dma_mask);
dir, *dev->dma_mask);
}
static void flush_unmaps(void)
......@@ -2860,13 +3086,16 @@ static void flush_unmaps(void)
/* On real hardware multiple invalidations are expensive */
if (cap_caching_mode(iommu->cap))
iommu_flush_iotlb_psi(iommu, domain->id,
iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1,
!deferred_flush[i].freelist[j], 0);
else {
mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
(uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
}
__free_iova(&deferred_flush[i].domain[j]->iovad, iova);
if (deferred_flush[i].freelist[j])
dma_free_pagelist(deferred_flush[i].freelist[j]);
}
deferred_flush[i].next = 0;
}
......@@ -2883,7 +3112,7 @@ static void flush_unmaps_timeout(unsigned long data)
spin_unlock_irqrestore(&async_umap_flush_lock, flags);
}
static void add_unmap(struct dmar_domain *dom, struct iova *iova)
static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
{
unsigned long flags;
int next, iommu_id;
......@@ -2899,6 +3128,7 @@ static void add_unmap(struct dmar_domain *dom, struct iova *iova)
next = deferred_flush[iommu_id].next;
deferred_flush[iommu_id].domain[next] = dom;
deferred_flush[iommu_id].iova[next] = iova;
deferred_flush[iommu_id].freelist[next] = freelist;
deferred_flush[iommu_id].next++;
if (!timer_on) {
......@@ -2913,16 +3143,16 @@ static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
size_t size, enum dma_data_direction dir,
struct dma_attrs *attrs)
{
struct pci_dev *pdev = to_pci_dev(dev);
struct dmar_domain *domain;
unsigned long start_pfn, last_pfn;
struct iova *iova;
struct intel_iommu *iommu;
struct page *freelist;
if (iommu_no_mapping(dev))
return;
domain = find_domain(pdev);
domain = find_domain(dev);
BUG_ON(!domain);
iommu = domain_get_iommu(domain);
......@@ -2936,21 +3166,18 @@ static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
pr_debug("Device %s unmapping: pfn %lx-%lx\n",
pci_name(pdev), start_pfn, last_pfn);
/* clear the whole page */
dma_pte_clear_range(domain, start_pfn, last_pfn);
dev_name(dev), start_pfn, last_pfn);
/* free page tables */
dma_pte_free_pagetable(domain, start_pfn, last_pfn);
freelist = domain_unmap(domain, start_pfn, last_pfn);
if (intel_iommu_strict) {
iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
last_pfn - start_pfn + 1, 0);
last_pfn - start_pfn + 1, !freelist, 0);
/* free iova */
__free_iova(&domain->iovad, iova);
dma_free_pagelist(freelist);
} else {
add_unmap(domain, iova);
add_unmap(domain, iova, freelist);
/*
* queue up the release of the unmap to save the 1/6th of the
* cpu used up by the iotlb flush operation...
......@@ -2958,7 +3185,7 @@ static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
}
}
static void *intel_alloc_coherent(struct device *hwdev, size_t size,
static void *intel_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
struct dma_attrs *attrs)
{
......@@ -2968,10 +3195,10 @@ static void *intel_alloc_coherent(struct device *hwdev, size_t size,
size = PAGE_ALIGN(size);
order = get_order(size);
if (!iommu_no_mapping(hwdev))
if (!iommu_no_mapping(dev))
flags &= ~(GFP_DMA | GFP_DMA32);
else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
flags |= GFP_DMA;
else
flags |= GFP_DMA32;
......@@ -2982,16 +3209,16 @@ static void *intel_alloc_coherent(struct device *hwdev, size_t size,
return NULL;
memset(vaddr, 0, size);
*dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
*dma_handle = __intel_map_single(dev, virt_to_bus(vaddr), size,
DMA_BIDIRECTIONAL,
hwdev->coherent_dma_mask);
dev->coherent_dma_mask);
if (*dma_handle)
return vaddr;
free_pages((unsigned long)vaddr, order);
return NULL;
}
static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_handle, struct dma_attrs *attrs)
{
int order;
......@@ -2999,24 +3226,24 @@ static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
size = PAGE_ALIGN(size);
order = get_order(size);
intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
intel_unmap_page(dev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
free_pages((unsigned long)vaddr, order);
}
static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
int nelems, enum dma_data_direction dir,
struct dma_attrs *attrs)
{
struct pci_dev *pdev = to_pci_dev(hwdev);
struct dmar_domain *domain;
unsigned long start_pfn, last_pfn;
struct iova *iova;
struct intel_iommu *iommu;
struct page *freelist;
if (iommu_no_mapping(hwdev))
if (iommu_no_mapping(dev))
return;
domain = find_domain(pdev);
domain = find_domain(dev);
BUG_ON(!domain);
iommu = domain_get_iommu(domain);
......@@ -3029,19 +3256,16 @@ static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
start_pfn = mm_to_dma_pfn(iova->pfn_lo);
last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
/* clear the whole page */
dma_pte_clear_range(domain, start_pfn, last_pfn);
/* free page tables */
dma_pte_free_pagetable(domain, start_pfn, last_pfn);
freelist = domain_unmap(domain, start_pfn, last_pfn);
if (intel_iommu_strict) {
iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
last_pfn - start_pfn + 1, 0);
last_pfn - start_pfn + 1, !freelist, 0);
/* free iova */
__free_iova(&domain->iovad, iova);
dma_free_pagelist(freelist);
} else {
add_unmap(domain, iova);
add_unmap(domain, iova, freelist);
/*
* queue up the release of the unmap to save the 1/6th of the
* cpu used up by the iotlb flush operation...
......@@ -3063,11 +3287,10 @@ static int intel_nontranslate_map_sg(struct device *hddev,
return nelems;
}
static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
enum dma_data_direction dir, struct dma_attrs *attrs)
{
int i;
struct pci_dev *pdev = to_pci_dev(hwdev);
struct dmar_domain *domain;
size_t size = 0;
int prot = 0;
......@@ -3078,10 +3301,10 @@ static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int ne
struct intel_iommu *iommu;
BUG_ON(dir == DMA_NONE);
if (iommu_no_mapping(hwdev))
return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
if (iommu_no_mapping(dev))
return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
domain = get_valid_domain_for_dev(pdev);
domain = get_valid_domain_for_dev(dev);
if (!domain)
return 0;
......@@ -3090,8 +3313,8 @@ static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int ne
for_each_sg(sglist, sg, nelems, i)
size += aligned_nrpages(sg->offset, sg->length);
iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
pdev->dma_mask);
iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
*dev->dma_mask);
if (!iova) {
sglist->dma_length = 0;
return 0;
......@@ -3124,7 +3347,7 @@ static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int ne
/* it's a non-present to present mapping. Only flush if caching mode */
if (cap_caching_mode(iommu->cap))
iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
else
iommu_flush_write_buffer(iommu);
......@@ -3259,29 +3482,28 @@ DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quir
static void __init init_no_remapping_devices(void)
{
struct dmar_drhd_unit *drhd;
struct device *dev;
int i;
for_each_drhd_unit(drhd) {
if (!drhd->include_all) {
int i;
for (i = 0; i < drhd->devices_cnt; i++)
if (drhd->devices[i] != NULL)
break;
/* ignore DMAR unit if no pci devices exist */
for_each_active_dev_scope(drhd->devices,
drhd->devices_cnt, i, dev)
break;
/* ignore DMAR unit if no devices exist */
if (i == drhd->devices_cnt)
drhd->ignored = 1;
}
}
for_each_active_drhd_unit(drhd) {
int i;
if (drhd->include_all)
continue;
for (i = 0; i < drhd->devices_cnt; i++)
if (drhd->devices[i] &&
!IS_GFX_DEVICE(drhd->devices[i]))
for_each_active_dev_scope(drhd->devices,
drhd->devices_cnt, i, dev)
if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
break;
if (i < drhd->devices_cnt)
continue;
......@@ -3291,11 +3513,9 @@ static void __init init_no_remapping_devices(void)
intel_iommu_gfx_mapped = 1;
} else {
drhd->ignored = 1;
for (i = 0; i < drhd->devices_cnt; i++) {
if (!drhd->devices[i])
continue;
drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
}
for_each_active_dev_scope(drhd->devices,
drhd->devices_cnt, i, dev)
dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
}
}
}
......@@ -3438,13 +3658,6 @@ static void __init init_iommu_pm_ops(void)
static inline void init_iommu_pm_ops(void) {}
#endif /* CONFIG_PM */
LIST_HEAD(dmar_rmrr_units);
static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
{
list_add(&rmrr->list, &dmar_rmrr_units);
}
int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
{
......@@ -3459,25 +3672,19 @@ int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
rmrr = (struct acpi_dmar_reserved_memory *)header;
rmrru->base_address = rmrr->base_address;
rmrru->end_address = rmrr->end_address;
rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
((void *)rmrr) + rmrr->header.length,
&rmrru->devices_cnt);
if (rmrru->devices_cnt && rmrru->devices == NULL) {
kfree(rmrru);
return -ENOMEM;
}
dmar_register_rmrr_unit(rmrru);
return 0;
}
static int __init
rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
{
struct acpi_dmar_reserved_memory *rmrr;
list_add(&rmrru->list, &dmar_rmrr_units);
rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
return dmar_parse_dev_scope((void *)(rmrr + 1),
((void *)rmrr) + rmrr->header.length,
&rmrru->devices_cnt, &rmrru->devices,
rmrr->segment);
return 0;
}
static LIST_HEAD(dmar_atsr_units);
int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
{
struct acpi_dmar_atsr *atsr;
......@@ -3490,26 +3697,21 @@ int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
atsru->hdr = hdr;
atsru->include_all = atsr->flags & 0x1;
if (!atsru->include_all) {
atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
(void *)atsr + atsr->header.length,
&atsru->devices_cnt);
if (atsru->devices_cnt && atsru->devices == NULL) {
kfree(atsru);
return -ENOMEM;
}
}
list_add(&atsru->list, &dmar_atsr_units);
list_add_rcu(&atsru->list, &dmar_atsr_units);
return 0;
}
static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
{
struct acpi_dmar_atsr *atsr;
if (atsru->include_all)
return 0;
atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
return dmar_parse_dev_scope((void *)(atsr + 1),
(void *)atsr + atsr->header.length,
&atsru->devices_cnt, &atsru->devices,
atsr->segment);
}
static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
{
dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
......@@ -3535,62 +3737,97 @@ static void intel_iommu_free_dmars(void)
int dmar_find_matched_atsr_unit(struct pci_dev *dev)
{
int i;
int i, ret = 1;
struct pci_bus *bus;
struct pci_dev *bridge = NULL;
struct device *tmp;
struct acpi_dmar_atsr *atsr;
struct dmar_atsr_unit *atsru;
dev = pci_physfn(dev);
list_for_each_entry(atsru, &dmar_atsr_units, list) {
atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
if (atsr->segment == pci_domain_nr(dev->bus))
goto found;
}
return 0;
found:
for (bus = dev->bus; bus; bus = bus->parent) {
struct pci_dev *bridge = bus->self;
bridge = bus->self;
if (!bridge || !pci_is_pcie(bridge) ||
pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
return 0;
if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT) {
for (i = 0; i < atsru->devices_cnt; i++)
if (atsru->devices[i] == bridge)
return 1;
if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
break;
}
}
if (!bridge)
return 0;
if (atsru->include_all)
return 1;
rcu_read_lock();
list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
if (atsr->segment != pci_domain_nr(dev->bus))
continue;
return 0;
for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
if (tmp == &bridge->dev)
goto out;
if (atsru->include_all)
goto out;
}
ret = 0;
out:
rcu_read_unlock();
return ret;
}
int __init dmar_parse_rmrr_atsr_dev(void)
int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
{
struct dmar_rmrr_unit *rmrr;
struct dmar_atsr_unit *atsr;
int ret = 0;
struct dmar_rmrr_unit *rmrru;
struct dmar_atsr_unit *atsru;
struct acpi_dmar_atsr *atsr;
struct acpi_dmar_reserved_memory *rmrr;
list_for_each_entry(rmrr, &dmar_rmrr_units, list) {
ret = rmrr_parse_dev(rmrr);
if (ret)
return ret;
if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
return 0;
list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
rmrr = container_of(rmrru->hdr,
struct acpi_dmar_reserved_memory, header);
if (info->event == BUS_NOTIFY_ADD_DEVICE) {
ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
((void *)rmrr) + rmrr->header.length,
rmrr->segment, rmrru->devices,
rmrru->devices_cnt);
if (ret > 0)
break;
else if(ret < 0)
return ret;
} else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
if (dmar_remove_dev_scope(info, rmrr->segment,
rmrru->devices, rmrru->devices_cnt))
break;
}
}
list_for_each_entry(atsr, &dmar_atsr_units, list) {
ret = atsr_parse_dev(atsr);
if (ret)
return ret;
list_for_each_entry(atsru, &dmar_atsr_units, list) {
if (atsru->include_all)
continue;
atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
if (info->event == BUS_NOTIFY_ADD_DEVICE) {
ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
(void *)atsr + atsr->header.length,
atsr->segment, atsru->devices,
atsru->devices_cnt);
if (ret > 0)
break;
else if(ret < 0)
return ret;
} else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
if (dmar_remove_dev_scope(info, atsr->segment,
atsru->devices, atsru->devices_cnt))
break;
}
}
return ret;
return 0;
}
/*
......@@ -3603,24 +3840,26 @@ static int device_notifier(struct notifier_block *nb,
unsigned long action, void *data)
{
struct device *dev = data;
struct pci_dev *pdev = to_pci_dev(dev);
struct dmar_domain *domain;
if (iommu_no_mapping(dev))
if (iommu_dummy(dev))
return 0;
domain = find_domain(pdev);
if (!domain)
if (action != BUS_NOTIFY_UNBOUND_DRIVER &&
action != BUS_NOTIFY_DEL_DEVICE)
return 0;
if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
domain_remove_one_dev_info(domain, pdev);
domain = find_domain(dev);
if (!domain)
return 0;
if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
!(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
list_empty(&domain->devices))
domain_exit(domain);
}
down_read(&dmar_global_lock);
domain_remove_one_dev_info(domain, dev);
if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
!(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
list_empty(&domain->devices))
domain_exit(domain);
up_read(&dmar_global_lock);
return 0;
}
......@@ -3629,6 +3868,75 @@ static struct notifier_block device_nb = {
.notifier_call = device_notifier,
};
static int intel_iommu_memory_notifier(struct notifier_block *nb,
unsigned long val, void *v)
{
struct memory_notify *mhp = v;
unsigned long long start, end;
unsigned long start_vpfn, last_vpfn;
switch (val) {
case MEM_GOING_ONLINE:
start = mhp->start_pfn << PAGE_SHIFT;
end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
if (iommu_domain_identity_map(si_domain, start, end)) {
pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
start, end);
return NOTIFY_BAD;
}
break;
case MEM_OFFLINE:
case MEM_CANCEL_ONLINE:
start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
while (start_vpfn <= last_vpfn) {
struct iova *iova;
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
struct page *freelist;
iova = find_iova(&si_domain->iovad, start_vpfn);
if (iova == NULL) {
pr_debug("dmar: failed get IOVA for PFN %lx\n",
start_vpfn);
break;
}
iova = split_and_remove_iova(&si_domain->iovad, iova,
start_vpfn, last_vpfn);
if (iova == NULL) {
pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
start_vpfn, last_vpfn);
return NOTIFY_BAD;
}
freelist = domain_unmap(si_domain, iova->pfn_lo,
iova->pfn_hi);
rcu_read_lock();
for_each_active_iommu(iommu, drhd)
iommu_flush_iotlb_psi(iommu, si_domain->id,
iova->pfn_lo,
iova->pfn_hi - iova->pfn_lo + 1,
!freelist, 0);
rcu_read_unlock();
dma_free_pagelist(freelist);
start_vpfn = iova->pfn_hi + 1;
free_iova_mem(iova);
}
break;
}
return NOTIFY_OK;
}
static struct notifier_block intel_iommu_memory_nb = {
.notifier_call = intel_iommu_memory_notifier,
.priority = 0
};
int __init intel_iommu_init(void)
{
int ret = -ENODEV;
......@@ -3638,6 +3946,13 @@ int __init intel_iommu_init(void)
/* VT-d is required for a TXT/tboot launch, so enforce that */
force_on = tboot_force_iommu();
if (iommu_init_mempool()) {
if (force_on)
panic("tboot: Failed to initialize iommu memory\n");
return -ENOMEM;
}
down_write(&dmar_global_lock);
if (dmar_table_init()) {
if (force_on)
panic("tboot: Failed to initialize DMAR table\n");
......@@ -3660,12 +3975,6 @@ int __init intel_iommu_init(void)
if (no_iommu || dmar_disabled)
goto out_free_dmar;
if (iommu_init_mempool()) {
if (force_on)
panic("tboot: Failed to initialize iommu memory\n");
goto out_free_dmar;
}
if (list_empty(&dmar_rmrr_units))
printk(KERN_INFO "DMAR: No RMRR found\n");
......@@ -3675,7 +3984,7 @@ int __init intel_iommu_init(void)
if (dmar_init_reserved_ranges()) {
if (force_on)
panic("tboot: Failed to reserve iommu ranges\n");
goto out_free_mempool;
goto out_free_reserved_range;
}
init_no_remapping_devices();
......@@ -3687,6 +3996,7 @@ int __init intel_iommu_init(void)
printk(KERN_ERR "IOMMU: dmar init failed\n");
goto out_free_reserved_range;
}
up_write(&dmar_global_lock);
printk(KERN_INFO
"PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
......@@ -3699,8 +4009,9 @@ int __init intel_iommu_init(void)
init_iommu_pm_ops();
bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
bus_register_notifier(&pci_bus_type, &device_nb);
if (si_domain && !hw_pass_through)
register_memory_notifier(&intel_iommu_memory_nb);
intel_iommu_enabled = 1;
......@@ -3708,21 +4019,23 @@ int __init intel_iommu_init(void)
out_free_reserved_range:
put_iova_domain(&reserved_iova_list);
out_free_mempool:
iommu_exit_mempool();
out_free_dmar:
intel_iommu_free_dmars();
up_write(&dmar_global_lock);
iommu_exit_mempool();
return ret;
}
static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
struct pci_dev *pdev)
struct device *dev)
{
struct pci_dev *tmp, *parent;
struct pci_dev *tmp, *parent, *pdev;
if (!iommu || !pdev)
if (!iommu || !dev || !dev_is_pci(dev))
return;
pdev = to_pci_dev(dev);
/* dependent device detach */
tmp = pci_find_upstream_pcie_bridge(pdev);
/* Secondary interface's bus number and devfn 0 */
......@@ -3743,29 +4056,28 @@ static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
}
static void domain_remove_one_dev_info(struct dmar_domain *domain,
struct pci_dev *pdev)
struct device *dev)
{
struct device_domain_info *info, *tmp;
struct intel_iommu *iommu;
unsigned long flags;
int found = 0;
u8 bus, devfn;
iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
pdev->devfn);
iommu = device_to_iommu(dev, &bus, &devfn);
if (!iommu)
return;
spin_lock_irqsave(&device_domain_lock, flags);
list_for_each_entry_safe(info, tmp, &domain->devices, link) {
if (info->segment == pci_domain_nr(pdev->bus) &&
info->bus == pdev->bus->number &&
info->devfn == pdev->devfn) {
if (info->iommu == iommu && info->bus == bus &&
info->devfn == devfn) {
unlink_domain_info(info);
spin_unlock_irqrestore(&device_domain_lock, flags);
iommu_disable_dev_iotlb(info);
iommu_detach_dev(iommu, info->bus, info->devfn);
iommu_detach_dependent_devices(iommu, pdev);
iommu_detach_dependent_devices(iommu, dev);
free_devinfo_mem(info);
spin_lock_irqsave(&device_domain_lock, flags);
......@@ -3780,8 +4092,7 @@ static void domain_remove_one_dev_info(struct dmar_domain *domain,
* owned by this domain, clear this iommu in iommu_bmp
* update iommu count and coherency
*/
if (iommu == device_to_iommu(info->segment, info->bus,
info->devfn))
if (info->iommu == iommu)
found = 1;
}
......@@ -3805,67 +4116,11 @@ static void domain_remove_one_dev_info(struct dmar_domain *domain,
}
}
static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
{
struct device_domain_info *info;
struct intel_iommu *iommu;
unsigned long flags1, flags2;
spin_lock_irqsave(&device_domain_lock, flags1);
while (!list_empty(&domain->devices)) {
info = list_entry(domain->devices.next,
struct device_domain_info, link);
unlink_domain_info(info);
spin_unlock_irqrestore(&device_domain_lock, flags1);
iommu_disable_dev_iotlb(info);
iommu = device_to_iommu(info->segment, info->bus, info->devfn);
iommu_detach_dev(iommu, info->bus, info->devfn);
iommu_detach_dependent_devices(iommu, info->dev);
/* clear this iommu in iommu_bmp, update iommu count
* and capabilities
*/
spin_lock_irqsave(&domain->iommu_lock, flags2);
if (test_and_clear_bit(iommu->seq_id,
domain->iommu_bmp)) {
domain->iommu_count--;
domain_update_iommu_cap(domain);
}
spin_unlock_irqrestore(&domain->iommu_lock, flags2);
free_devinfo_mem(info);
spin_lock_irqsave(&device_domain_lock, flags1);
}
spin_unlock_irqrestore(&device_domain_lock, flags1);
}
/* domain id for virtual machine, it won't be set in context */
static atomic_t vm_domid = ATOMIC_INIT(0);
static struct dmar_domain *iommu_alloc_vm_domain(void)
{
struct dmar_domain *domain;
domain = alloc_domain_mem();
if (!domain)
return NULL;
domain->id = atomic_inc_return(&vm_domid);
domain->nid = -1;
memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
return domain;
}
static int md_domain_init(struct dmar_domain *domain, int guest_width)
{
int adjust_width;
init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
spin_lock_init(&domain->iommu_lock);
domain_reserve_special_ranges(domain);
/* calculate AGAW */
......@@ -3873,9 +4128,6 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width)
adjust_width = guestwidth_to_adjustwidth(guest_width);
domain->agaw = width_to_agaw(adjust_width);
INIT_LIST_HEAD(&domain->devices);
domain->iommu_count = 0;
domain->iommu_coherency = 0;
domain->iommu_snooping = 0;
domain->iommu_superpage = 0;
......@@ -3890,53 +4142,11 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width)
return 0;
}
static void iommu_free_vm_domain(struct dmar_domain *domain)
{
unsigned long flags;
struct dmar_drhd_unit *drhd;
struct intel_iommu *iommu;
unsigned long i;
unsigned long ndomains;
for_each_active_iommu(iommu, drhd) {
ndomains = cap_ndoms(iommu->cap);
for_each_set_bit(i, iommu->domain_ids, ndomains) {
if (iommu->domains[i] == domain) {
spin_lock_irqsave(&iommu->lock, flags);
clear_bit(i, iommu->domain_ids);
iommu->domains[i] = NULL;
spin_unlock_irqrestore(&iommu->lock, flags);
break;
}
}
}
}
static void vm_domain_exit(struct dmar_domain *domain)
{
/* Domain 0 is reserved, so dont process it */
if (!domain)
return;
vm_domain_remove_all_dev_info(domain);
/* destroy iovas */
put_iova_domain(&domain->iovad);
/* clear ptes */
dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
/* free page tables */
dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
iommu_free_vm_domain(domain);
free_domain_mem(domain);
}
static int intel_iommu_domain_init(struct iommu_domain *domain)
{
struct dmar_domain *dmar_domain;
dmar_domain = iommu_alloc_vm_domain();
dmar_domain = alloc_domain(true);
if (!dmar_domain) {
printk(KERN_ERR
"intel_iommu_domain_init: dmar_domain == NULL\n");
......@@ -3945,7 +4155,7 @@ static int intel_iommu_domain_init(struct iommu_domain *domain)
if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
printk(KERN_ERR
"intel_iommu_domain_init() failed\n");
vm_domain_exit(dmar_domain);
domain_exit(dmar_domain);
return -ENOMEM;
}
domain_update_iommu_cap(dmar_domain);
......@@ -3963,33 +4173,32 @@ static void intel_iommu_domain_destroy(struct iommu_domain *domain)
struct dmar_domain *dmar_domain = domain->priv;
domain->priv = NULL;
vm_domain_exit(dmar_domain);
domain_exit(dmar_domain);
}
static int intel_iommu_attach_device(struct iommu_domain *domain,
struct device *dev)
{
struct dmar_domain *dmar_domain = domain->priv;
struct pci_dev *pdev = to_pci_dev(dev);
struct intel_iommu *iommu;
int addr_width;
u8 bus, devfn;
/* normally pdev is not mapped */
if (unlikely(domain_context_mapped(pdev))) {
/* normally dev is not mapped */
if (unlikely(domain_context_mapped(dev))) {
struct dmar_domain *old_domain;
old_domain = find_domain(pdev);
old_domain = find_domain(dev);
if (old_domain) {
if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
domain_remove_one_dev_info(old_domain, pdev);
domain_remove_one_dev_info(old_domain, dev);
else
domain_remove_dev_info(old_domain);
}
}
iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
pdev->devfn);
iommu = device_to_iommu(dev, &bus, &devfn);
if (!iommu)
return -ENODEV;
......@@ -4021,16 +4230,15 @@ static int intel_iommu_attach_device(struct iommu_domain *domain,
dmar_domain->agaw--;
}
return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
}
static void intel_iommu_detach_device(struct iommu_domain *domain,
struct device *dev)
{
struct dmar_domain *dmar_domain = domain->priv;
struct pci_dev *pdev = to_pci_dev(dev);
domain_remove_one_dev_info(dmar_domain, pdev);
domain_remove_one_dev_info(dmar_domain, dev);
}
static int intel_iommu_map(struct iommu_domain *domain,
......@@ -4072,18 +4280,51 @@ static int intel_iommu_map(struct iommu_domain *domain,
}
static size_t intel_iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size)
unsigned long iova, size_t size)
{
struct dmar_domain *dmar_domain = domain->priv;
int order;
struct page *freelist = NULL;
struct intel_iommu *iommu;
unsigned long start_pfn, last_pfn;
unsigned int npages;
int iommu_id, num, ndomains, level = 0;
/* Cope with horrid API which requires us to unmap more than the
size argument if it happens to be a large-page mapping. */
if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
BUG();
if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
size = VTD_PAGE_SIZE << level_to_offset_bits(level);
start_pfn = iova >> VTD_PAGE_SHIFT;
last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
npages = last_pfn - start_pfn + 1;
for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
iommu = g_iommus[iommu_id];
/*
* find bit position of dmar_domain
*/
ndomains = cap_ndoms(iommu->cap);
for_each_set_bit(num, iommu->domain_ids, ndomains) {
if (iommu->domains[num] == dmar_domain)
iommu_flush_iotlb_psi(iommu, num, start_pfn,
npages, !freelist, 0);
}
}
order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
(iova + size - 1) >> VTD_PAGE_SHIFT);
dma_free_pagelist(freelist);
if (dmar_domain->max_addr == iova + size)
dmar_domain->max_addr = iova;
return PAGE_SIZE << order;
return size;
}
static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
......@@ -4091,9 +4332,10 @@ static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
{
struct dmar_domain *dmar_domain = domain->priv;
struct dma_pte *pte;
int level = 0;
u64 phys = 0;
pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
if (pte)
phys = dma_pte_addr(pte);
......@@ -4121,9 +4363,9 @@ static int intel_iommu_add_device(struct device *dev)
struct pci_dev *bridge, *dma_pdev = NULL;
struct iommu_group *group;
int ret;
u8 bus, devfn;
if (!device_to_iommu(pci_domain_nr(pdev->bus),
pdev->bus->number, pdev->devfn))
if (!device_to_iommu(dev, &bus, &devfn))
return -ENODEV;
bridge = pci_find_upstream_pcie_bridge(pdev);
......
drivers/iommu/intel_irq_remapping.c
View file @ 3f583bc2
......@@ -38,6 +38,17 @@ static struct ioapic_scope ir_ioapic[MAX_IO_APICS];
static struct hpet_scope ir_hpet[MAX_HPET_TBS];
static int ir_ioapic_num, ir_hpet_num;
/*
* Lock ordering:
* ->dmar_global_lock
* ->irq_2_ir_lock
* ->qi->q_lock
* ->iommu->register_lock
* Note:
* intel_irq_remap_ops.{supported,prepare,enable,disable,reenable} are called
* in single-threaded environment with interrupt disabled, so no need to tabke
* the dmar_global_lock.
*/
static DEFINE_RAW_SPINLOCK(irq_2_ir_lock);
static int __init parse_ioapics_under_ir(void);
......@@ -307,12 +318,14 @@ static int set_ioapic_sid(struct irte *irte, int apic)
if (!irte)
return -1;
down_read(&dmar_global_lock);
for (i = 0; i < MAX_IO_APICS; i++) {
if (ir_ioapic[i].id == apic) {
sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn;
break;
}
}
up_read(&dmar_global_lock);
if (sid == 0) {
pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic);
......@@ -332,12 +345,14 @@ static int set_hpet_sid(struct irte *irte, u8 id)
if (!irte)
return -1;
down_read(&dmar_global_lock);
for (i = 0; i < MAX_HPET_TBS; i++) {
if (ir_hpet[i].id == id) {
sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn;
break;
}
}
up_read(&dmar_global_lock);
if (sid == 0) {
pr_warning("Failed to set source-id of HPET block (%d)\n", id);
......@@ -794,10 +809,16 @@ static int __init parse_ioapics_under_ir(void)
static int __init ir_dev_scope_init(void)
{
int ret;
if (!irq_remapping_enabled)
return 0;
return dmar_dev_scope_init();
down_write(&dmar_global_lock);
ret = dmar_dev_scope_init();
up_write(&dmar_global_lock);
return ret;
}
rootfs_initcall(ir_dev_scope_init);
......@@ -878,23 +899,27 @@ static int intel_setup_ioapic_entry(int irq,
struct io_apic_irq_attr *attr)
{
int ioapic_id = mpc_ioapic_id(attr->ioapic);
struct intel_iommu *iommu = map_ioapic_to_ir(ioapic_id);
struct intel_iommu *iommu;
struct IR_IO_APIC_route_entry *entry;
struct irte irte;
int index;
down_read(&dmar_global_lock);
iommu = map_ioapic_to_ir(ioapic_id);
if (!iommu) {
pr_warn("No mapping iommu for ioapic %d\n", ioapic_id);
return -ENODEV;
}
entry = (struct IR_IO_APIC_route_entry *)route_entry;
index = alloc_irte(iommu, irq, 1);
if (index < 0) {
pr_warn("Failed to allocate IRTE for ioapic %d\n", ioapic_id);
return -ENOMEM;
index = -ENODEV;
} else {
index = alloc_irte(iommu, irq, 1);
if (index < 0) {
pr_warn("Failed to allocate IRTE for ioapic %d\n",
ioapic_id);
index = -ENOMEM;
}
}
up_read(&dmar_global_lock);
if (index < 0)
return index;
prepare_irte(&irte, vector, destination);
......@@ -913,6 +938,7 @@ static int intel_setup_ioapic_entry(int irq,
irte.avail, irte.vector, irte.dest_id,
irte.sid, irte.sq, irte.svt);
entry = (struct IR_IO_APIC_route_entry *)route_entry;
memset(entry, 0, sizeof(*entry));
entry->index2 = (index >> 15) & 0x1;
......@@ -1043,20 +1069,23 @@ static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec)
struct intel_iommu *iommu;
int index;
down_read(&dmar_global_lock);
iommu = map_dev_to_ir(dev);
if (!iommu) {
printk(KERN_ERR
"Unable to map PCI %s to iommu\n", pci_name(dev));
return -ENOENT;
index = -ENOENT;
} else {
index = alloc_irte(iommu, irq, nvec);
if (index < 0) {
printk(KERN_ERR
"Unable to allocate %d IRTE for PCI %s\n",
nvec, pci_name(dev));
index = -ENOSPC;
}
}
up_read(&dmar_global_lock);
index = alloc_irte(iommu, irq, nvec);
if (index < 0) {
printk(KERN_ERR
"Unable to allocate %d IRTE for PCI %s\n", nvec,
pci_name(dev));
return -ENOSPC;
}
return index;
}
......@@ -1064,33 +1093,40 @@ static int intel_msi_setup_irq(struct pci_dev *pdev, unsigned int irq,
int index, int sub_handle)
{
struct intel_iommu *iommu;
int ret = -ENOENT;
down_read(&dmar_global_lock);
iommu = map_dev_to_ir(pdev);
if (!iommu)
return -ENOENT;
/*
* setup the mapping between the irq and the IRTE
* base index, the sub_handle pointing to the
* appropriate interrupt remap table entry.
*/
set_irte_irq(irq, iommu, index, sub_handle);
if (iommu) {
/*
* setup the mapping between the irq and the IRTE
* base index, the sub_handle pointing to the
* appropriate interrupt remap table entry.
*/
set_irte_irq(irq, iommu, index, sub_handle);
ret = 0;
}
up_read(&dmar_global_lock);
return 0;
return ret;
}
static int intel_setup_hpet_msi(unsigned int irq, unsigned int id)
{
struct intel_iommu *iommu = map_hpet_to_ir(id);
int ret = -1;
struct intel_iommu *iommu;
int index;
if (!iommu)
return -1;
index = alloc_irte(iommu, irq, 1);
if (index < 0)
return -1;
down_read(&dmar_global_lock);
iommu = map_hpet_to_ir(id);
if (iommu) {
index = alloc_irte(iommu, irq, 1);
if (index >= 0)
ret = 0;
}
up_read(&dmar_global_lock);
return 0;
return ret;
}
struct irq_remap_ops intel_irq_remap_ops = {
......
drivers/iommu/iova.c
View file @ 3f583bc2
......@@ -342,19 +342,30 @@ __is_range_overlap(struct rb_node *node,
return 0;
}
static inline struct iova *
alloc_and_init_iova(unsigned long pfn_lo, unsigned long pfn_hi)
{
struct iova *iova;
iova = alloc_iova_mem();
if (iova) {
iova->pfn_lo = pfn_lo;
iova->pfn_hi = pfn_hi;
}
return iova;
}
static struct iova *
__insert_new_range(struct iova_domain *iovad,
unsigned long pfn_lo, unsigned long pfn_hi)
{
struct iova *iova;
iova = alloc_iova_mem();
if (!iova)
return iova;
iova = alloc_and_init_iova(pfn_lo, pfn_hi);
if (iova)
iova_insert_rbtree(&iovad->rbroot, iova);
iova->pfn_hi = pfn_hi;
iova->pfn_lo = pfn_lo;
iova_insert_rbtree(&iovad->rbroot, iova);
return iova;
}
......@@ -433,3 +444,44 @@ copy_reserved_iova(struct iova_domain *from, struct iova_domain *to)
}
spin_unlock_irqrestore(&from->iova_rbtree_lock, flags);
}
struct iova *
split_and_remove_iova(struct iova_domain *iovad, struct iova *iova,
unsigned long pfn_lo, unsigned long pfn_hi)
{
unsigned long flags;
struct iova *prev = NULL, *next = NULL;
spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
if (iova->pfn_lo < pfn_lo) {
prev = alloc_and_init_iova(iova->pfn_lo, pfn_lo - 1);
if (prev == NULL)
goto error;
}
if (iova->pfn_hi > pfn_hi) {
next = alloc_and_init_iova(pfn_hi + 1, iova->pfn_hi);
if (next == NULL)
goto error;
}
__cached_rbnode_delete_update(iovad, iova);
rb_erase(&iova->node, &iovad->rbroot);
if (prev) {
iova_insert_rbtree(&iovad->rbroot, prev);
iova->pfn_lo = pfn_lo;
}
if (next) {
iova_insert_rbtree(&iovad->rbroot, next);
iova->pfn_hi = pfn_hi;
}
spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
return iova;
error:
spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
if (prev)
free_iova_mem(prev);
return NULL;
}
drivers/iommu/omap-iommu.c
View file @ 3f583bc2
......@@ -23,6 +23,9 @@
#include <linux/spinlock.h>
#include <linux/io.h>
#include <linux/pm_runtime.h>
#include <linux/of.h>
#include <linux/of_iommu.h>
#include <linux/of_irq.h>
#include <asm/cacheflush.h>
......@@ -146,13 +149,10 @@ static int iommu_enable(struct omap_iommu *obj)
struct platform_device *pdev = to_platform_device(obj->dev);
struct iommu_platform_data *pdata = pdev->dev.platform_data;
if (!pdata)
return -EINVAL;
if (!arch_iommu)
return -ENODEV;
if (pdata->deassert_reset) {
if (pdata && pdata->deassert_reset) {
err = pdata->deassert_reset(pdev, pdata->reset_name);
if (err) {
dev_err(obj->dev, "deassert_reset failed: %d\n", err);
......@@ -172,14 +172,11 @@ static void iommu_disable(struct omap_iommu *obj)
struct platform_device *pdev = to_platform_device(obj->dev);
struct iommu_platform_data *pdata = pdev->dev.platform_data;
if (!pdata)
return;
arch_iommu->disable(obj);
pm_runtime_put_sync(obj->dev);
if (pdata->assert_reset)
if (pdata && pdata->assert_reset)
pdata->assert_reset(pdev, pdata->reset_name);
}
......@@ -523,7 +520,8 @@ static void flush_iopte_range(u32 *first, u32 *last)
static void iopte_free(u32 *iopte)
{
/* Note: freed iopte's must be clean ready for re-use */
kmem_cache_free(iopte_cachep, iopte);
if (iopte)
kmem_cache_free(iopte_cachep, iopte);
}
static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)
......@@ -863,7 +861,7 @@ static int device_match_by_alias(struct device *dev, void *data)
**/
static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd)
{
int err = -ENOMEM;
int err;
struct device *dev;
struct omap_iommu *obj;
......@@ -871,7 +869,7 @@ static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd)
(void *)name,
device_match_by_alias);
if (!dev)
return NULL;
return ERR_PTR(-ENODEV);
obj = to_iommu(dev);
......@@ -890,8 +888,10 @@ static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd)
goto err_enable;
flush_iotlb_all(obj);
if (!try_module_get(obj->owner))
if (!try_module_get(obj->owner)) {
err = -ENODEV;
goto err_module;
}
spin_unlock(&obj->iommu_lock);
......@@ -940,17 +940,41 @@ static int omap_iommu_probe(struct platform_device *pdev)
struct omap_iommu *obj;
struct resource *res;
struct iommu_platform_data *pdata = pdev->dev.platform_data;
struct device_node *of = pdev->dev.of_node;
obj = kzalloc(sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL);
obj = devm_kzalloc(&pdev->dev, sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL);
if (!obj)
return -ENOMEM;
obj->nr_tlb_entries = pdata->nr_tlb_entries;
obj->name = pdata->name;
if (of) {
obj->name = dev_name(&pdev->dev);
obj->nr_tlb_entries = 32;
err = of_property_read_u32(of, "ti,#tlb-entries",
&obj->nr_tlb_entries);
if (err && err != -EINVAL)
return err;
if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8)
return -EINVAL;
/*
* da_start and da_end are needed for omap-iovmm, so hardcode
* these values as used by OMAP3 ISP - the only user for
* omap-iovmm
*/
obj->da_start = 0;
obj->da_end = 0xfffff000;
if (of_find_property(of, "ti,iommu-bus-err-back", NULL))
obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN;
} else {
obj->nr_tlb_entries = pdata->nr_tlb_entries;
obj->name = pdata->name;
obj->da_start = pdata->da_start;
obj->da_end = pdata->da_end;
}
if (obj->da_end <= obj->da_start)
return -EINVAL;
obj->dev = &pdev->dev;
obj->ctx = (void *)obj + sizeof(*obj);
obj->da_start = pdata->da_start;
obj->da_end = pdata->da_end;
spin_lock_init(&obj->iommu_lock);
mutex_init(&obj->mmap_lock);
......@@ -958,33 +982,18 @@ static int omap_iommu_probe(struct platform_device *pdev)
INIT_LIST_HEAD(&obj->mmap);
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (!res) {
err = -ENODEV;
goto err_mem;
}
res = request_mem_region(res->start, resource_size(res),
dev_name(&pdev->dev));
if (!res) {
err = -EIO;
goto err_mem;
}
obj->regbase = ioremap(res->start, resource_size(res));
if (!obj->regbase) {
err = -ENOMEM;
goto err_ioremap;
}
obj->regbase = devm_ioremap_resource(obj->dev, res);
if (IS_ERR(obj->regbase))
return PTR_ERR(obj->regbase);
irq = platform_get_irq(pdev, 0);
if (irq < 0) {
err = -ENODEV;
goto err_irq;
}
err = request_irq(irq, iommu_fault_handler, IRQF_SHARED,
dev_name(&pdev->dev), obj);
if (irq < 0)
return -ENODEV;
err = devm_request_irq(obj->dev, irq, iommu_fault_handler, IRQF_SHARED,
dev_name(obj->dev), obj);
if (err < 0)
goto err_irq;
return err;
platform_set_drvdata(pdev, obj);
pm_runtime_irq_safe(obj->dev);
......@@ -992,42 +1001,34 @@ static int omap_iommu_probe(struct platform_device *pdev)
dev_info(&pdev->dev, "%s registered\n", obj->name);
return 0;
err_irq:
iounmap(obj->regbase);
err_ioremap:
release_mem_region(res->start, resource_size(res));
err_mem:
kfree(obj);
return err;
}
static int omap_iommu_remove(struct platform_device *pdev)
{
int irq;
struct resource *res;
struct omap_iommu *obj = platform_get_drvdata(pdev);
iopgtable_clear_entry_all(obj);
irq = platform_get_irq(pdev, 0);
free_irq(irq, obj);
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
release_mem_region(res->start, resource_size(res));
iounmap(obj->regbase);
pm_runtime_disable(obj->dev);
dev_info(&pdev->dev, "%s removed\n", obj->name);
kfree(obj);
return 0;
}
static struct of_device_id omap_iommu_of_match[] = {
{ .compatible = "ti,omap2-iommu" },
{ .compatible = "ti,omap4-iommu" },
{ .compatible = "ti,dra7-iommu" },
{},
};
MODULE_DEVICE_TABLE(of, omap_iommu_of_match);
static struct platform_driver omap_iommu_driver = {
.probe = omap_iommu_probe,
.remove = omap_iommu_remove,
.driver = {
.name = "omap-iommu",
.of_match_table = of_match_ptr(omap_iommu_of_match),
},
};
......@@ -1253,6 +1254,49 @@ static int omap_iommu_domain_has_cap(struct iommu_domain *domain,
return 0;
}
static int omap_iommu_add_device(struct device *dev)
{
struct omap_iommu_arch_data *arch_data;
struct device_node *np;
/*
* Allocate the archdata iommu structure for DT-based devices.
*
* TODO: Simplify this when removing non-DT support completely from the
* IOMMU users.
*/
if (!dev->of_node)
return 0;
np = of_parse_phandle(dev->of_node, "iommus", 0);
if (!np)
return 0;
arch_data = kzalloc(sizeof(*arch_data), GFP_KERNEL);
if (!arch_data) {
of_node_put(np);
return -ENOMEM;
}
arch_data->name = kstrdup(dev_name(dev), GFP_KERNEL);
dev->archdata.iommu = arch_data;
of_node_put(np);
return 0;
}
static void omap_iommu_remove_device(struct device *dev)
{
struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
if (!dev->of_node || !arch_data)
return;
kfree(arch_data->name);
kfree(arch_data);
}
static struct iommu_ops omap_iommu_ops = {
.domain_init = omap_iommu_domain_init,
.domain_destroy = omap_iommu_domain_destroy,
......@@ -1262,6 +1306,8 @@ static struct iommu_ops omap_iommu_ops = {
.unmap = omap_iommu_unmap,
.iova_to_phys = omap_iommu_iova_to_phys,
.domain_has_cap = omap_iommu_domain_has_cap,
.add_device = omap_iommu_add_device,
.remove_device = omap_iommu_remove_device,
.pgsize_bitmap = OMAP_IOMMU_PGSIZES,
};
......
drivers/iommu/omap-iommu.h
View file @ 3f583bc2
......@@ -52,6 +52,8 @@ struct omap_iommu {
void *ctx; /* iommu context: registres saved area */
u32 da_start;
u32 da_end;
int has_bus_err_back;
};
struct cr_regs {
......@@ -130,6 +132,7 @@ static inline struct omap_iommu *dev_to_omap_iommu(struct device *dev)
#define MMU_READ_CAM 0x68
#define MMU_READ_RAM 0x6c
#define MMU_EMU_FAULT_AD 0x70
#define MMU_GP_REG 0x88
#define MMU_REG_SIZE 256
......@@ -163,6 +166,8 @@ static inline struct omap_iommu *dev_to_omap_iommu(struct device *dev)
#define MMU_RAM_MIXED_MASK (1 << MMU_RAM_MIXED_SHIFT)
#define MMU_RAM_MIXED MMU_RAM_MIXED_MASK
#define MMU_GP_REG_BUS_ERR_BACK_EN 0x1
/*
* utilities for super page(16MB, 1MB, 64KB and 4KB)
*/
......
drivers/iommu/omap-iommu2.c
View file @ 3f583bc2
......@@ -98,6 +98,9 @@ static int omap2_iommu_enable(struct omap_iommu *obj)
iommu_write_reg(obj, pa, MMU_TTB);
if (obj->has_bus_err_back)
iommu_write_reg(obj, MMU_GP_REG_BUS_ERR_BACK_EN, MMU_GP_REG);
__iommu_set_twl(obj, true);
return 0;
......
include/acpi/actbl2.h
View file @ 3f583bc2
......@@ -424,7 +424,8 @@ enum acpi_dmar_type {
ACPI_DMAR_TYPE_RESERVED_MEMORY = 1,
ACPI_DMAR_TYPE_ATSR = 2,
ACPI_DMAR_HARDWARE_AFFINITY = 3,
ACPI_DMAR_TYPE_RESERVED = 4 /* 4 and greater are reserved */
ACPI_DMAR_TYPE_ANDD = 4,
ACPI_DMAR_TYPE_RESERVED = 5 /* 5 and greater are reserved */
};
/* DMAR Device Scope structure */
......@@ -445,7 +446,8 @@ enum acpi_dmar_scope_type {
ACPI_DMAR_SCOPE_TYPE_BRIDGE = 2,
ACPI_DMAR_SCOPE_TYPE_IOAPIC = 3,
ACPI_DMAR_SCOPE_TYPE_HPET = 4,
ACPI_DMAR_SCOPE_TYPE_RESERVED = 5 /* 5 and greater are reserved */
ACPI_DMAR_SCOPE_TYPE_ACPI = 5,
ACPI_DMAR_SCOPE_TYPE_RESERVED = 6 /* 6 and greater are reserved */
};
struct acpi_dmar_pci_path {
......@@ -507,6 +509,15 @@ struct acpi_dmar_rhsa {
u32 proximity_domain;
};
/* 4: ACPI Namespace Device Declaration Structure */
struct acpi_dmar_andd {
struct acpi_dmar_header header;
u8 reserved[3];
u8 device_number;
u8 object_name[];
};
/*******************************************************************************
*
* HPET - High Precision Event Timer table
......
include/linux/dmar.h
View file @ 3f583bc2
......@@ -25,6 +25,8 @@
#include <linux/types.h>
#include <linux/msi.h>
#include <linux/irqreturn.h>
#include <linux/rwsem.h>
#include <linux/rcupdate.h>
struct acpi_dmar_header;
......@@ -34,13 +36,19 @@ struct acpi_dmar_header;
struct intel_iommu;
struct dmar_dev_scope {
struct device __rcu *dev;
u8 bus;
u8 devfn;
};
#ifdef CONFIG_DMAR_TABLE
extern struct acpi_table_header *dmar_tbl;
struct dmar_drhd_unit {
struct list_head list; /* list of drhd units */
struct acpi_dmar_header *hdr; /* ACPI header */
u64 reg_base_addr; /* register base address*/
struct pci_dev **devices; /* target device array */
struct dmar_dev_scope *devices;/* target device array */
int devices_cnt; /* target device count */
u16 segment; /* PCI domain */
u8 ignored:1; /* ignore drhd */
......@@ -48,33 +56,66 @@ struct dmar_drhd_unit {
struct intel_iommu *iommu;
};
struct dmar_pci_notify_info {
struct pci_dev *dev;
unsigned long event;
int bus;
u16 seg;
u16 level;
struct acpi_dmar_pci_path path[];
} __attribute__((packed));
extern struct rw_semaphore dmar_global_lock;
extern struct list_head dmar_drhd_units;
#define for_each_drhd_unit(drhd) \
list_for_each_entry(drhd, &dmar_drhd_units, list)
list_for_each_entry_rcu(drhd, &dmar_drhd_units, list)
#define for_each_active_drhd_unit(drhd) \
list_for_each_entry(drhd, &dmar_drhd_units, list) \
list_for_each_entry_rcu(drhd, &dmar_drhd_units, list) \
if (drhd->ignored) {} else
#define for_each_active_iommu(i, drhd) \
list_for_each_entry(drhd, &dmar_drhd_units, list) \
list_for_each_entry_rcu(drhd, &dmar_drhd_units, list) \
if (i=drhd->iommu, drhd->ignored) {} else
#define for_each_iommu(i, drhd) \
list_for_each_entry(drhd, &dmar_drhd_units, list) \
list_for_each_entry_rcu(drhd, &dmar_drhd_units, list) \
if (i=drhd->iommu, 0) {} else
static inline bool dmar_rcu_check(void)
{
return rwsem_is_locked(&dmar_global_lock) ||
system_state == SYSTEM_BOOTING;
}
#define dmar_rcu_dereference(p) rcu_dereference_check((p), dmar_rcu_check())
#define for_each_dev_scope(a, c, p, d) \
for ((p) = 0; ((d) = (p) < (c) ? dmar_rcu_dereference((a)[(p)].dev) : \
NULL, (p) < (c)); (p)++)
#define for_each_active_dev_scope(a, c, p, d) \
for_each_dev_scope((a), (c), (p), (d)) if (!(d)) { continue; } else
extern int dmar_table_init(void);
extern int dmar_dev_scope_init(void);
extern int dmar_parse_dev_scope(void *start, void *end, int *cnt,
struct pci_dev ***devices, u16 segment);
extern void dmar_free_dev_scope(struct pci_dev ***devices, int *cnt);
struct dmar_dev_scope **devices, u16 segment);
extern void *dmar_alloc_dev_scope(void *start, void *end, int *cnt);
extern void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt);
extern int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
void *start, void*end, u16 segment,
struct dmar_dev_scope *devices,
int devices_cnt);
extern int dmar_remove_dev_scope(struct dmar_pci_notify_info *info,
u16 segment, struct dmar_dev_scope *devices,
int count);
/* Intel IOMMU detection */
extern int detect_intel_iommu(void);
extern int enable_drhd_fault_handling(void);
#else
struct dmar_pci_notify_info;
static inline int detect_intel_iommu(void)
{
return -ENODEV;
......@@ -138,30 +179,9 @@ extern int arch_setup_dmar_msi(unsigned int irq);
#ifdef CONFIG_INTEL_IOMMU
extern int iommu_detected, no_iommu;
extern struct list_head dmar_rmrr_units;
struct dmar_rmrr_unit {
struct list_head list; /* list of rmrr units */
struct acpi_dmar_header *hdr; /* ACPI header */
u64 base_address; /* reserved base address*/
u64 end_address; /* reserved end address */
struct pci_dev **devices; /* target devices */
int devices_cnt; /* target device count */
};
#define for_each_rmrr_units(rmrr) \
list_for_each_entry(rmrr, &dmar_rmrr_units, list)
struct dmar_atsr_unit {
struct list_head list; /* list of ATSR units */
struct acpi_dmar_header *hdr; /* ACPI header */
struct pci_dev **devices; /* target devices */
int devices_cnt; /* target device count */
u8 include_all:1; /* include all ports */
};
int dmar_parse_rmrr_atsr_dev(void);
extern int dmar_parse_one_rmrr(struct acpi_dmar_header *header);
extern int dmar_parse_one_atsr(struct acpi_dmar_header *header);
extern int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info);
extern int intel_iommu_init(void);
#else /* !CONFIG_INTEL_IOMMU: */
static inline int intel_iommu_init(void) { return -ENODEV; }
......@@ -173,7 +193,7 @@ static inline int dmar_parse_one_atsr(struct acpi_dmar_header *header)
{
return 0;
}
static inline int dmar_parse_rmrr_atsr_dev(void)
static inline int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
{
return 0;
}
......
include/linux/intel-iommu.h
View file @ 3f583bc2
......@@ -319,6 +319,7 @@ struct intel_iommu {
int agaw; /* agaw of this iommu */
int msagaw; /* max sagaw of this iommu */
unsigned int irq;
u16 segment; /* PCI segment# */
unsigned char name[13]; /* Device Name */
#ifdef CONFIG_INTEL_IOMMU
......
include/linux/iova.h
View file @ 3f583bc2
......@@ -47,5 +47,7 @@ void copy_reserved_iova(struct iova_domain *from, struct iova_domain *to);
void init_iova_domain(struct iova_domain *iovad, unsigned long pfn_32bit);
struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
void put_iova_domain(struct iova_domain *iovad);
struct iova *split_and_remove_iova(struct iova_domain *iovad,
struct iova *iova, unsigned long pfn_lo, unsigned long pfn_hi);
#endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7