Commit c44ef998 authored by Ilya Lesokhin's avatar Ilya Lesokhin Committed by Doug Ledford

IB/mlx5: Maintain a single emergency page

The mlx5 driver needs to be able to issue invalidation to ODP MRs
even if it cannot allocate memory. To this end it preallocates
emergency pages to use when the situation arises.

This flow should be extremely rare enough, that we don't need
to worry about contention and therefore a single emergency page
is good enough.
Signed-off-by: default avatarIlya Lesokhin <ilyal@mellanox.com>
Signed-off-by: default avatarLeon Romanovsky <leon@kernel.org>
Signed-off-by: default avatarDoug Ledford <dledford@redhat.com>
parent 65edd0e7
...@@ -92,6 +92,12 @@ static LIST_HEAD(mlx5_ib_dev_list); ...@@ -92,6 +92,12 @@ static LIST_HEAD(mlx5_ib_dev_list);
*/ */
static DEFINE_MUTEX(mlx5_ib_multiport_mutex); static DEFINE_MUTEX(mlx5_ib_multiport_mutex);
/* We can't use an array for xlt_emergency_page because dma_map_single
* doesn't work on kernel modules memory
*/
static unsigned long xlt_emergency_page;
static struct mutex xlt_emergency_page_mutex;
struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi) struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)
{ {
struct mlx5_ib_dev *dev; struct mlx5_ib_dev *dev;
...@@ -1698,17 +1704,10 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev, ...@@ -1698,17 +1704,10 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
context->ibucontext.invalidate_range = &mlx5_ib_invalidate_range; context->ibucontext.invalidate_range = &mlx5_ib_invalidate_range;
#endif #endif
context->upd_xlt_page = __get_free_page(GFP_KERNEL);
if (!context->upd_xlt_page) {
err = -ENOMEM;
goto out_uars;
}
mutex_init(&context->upd_xlt_page_mutex);
if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain)) { if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain)) {
err = mlx5_ib_alloc_transport_domain(dev, &context->tdn); err = mlx5_ib_alloc_transport_domain(dev, &context->tdn);
if (err) if (err)
goto out_page; goto out_uars;
} }
INIT_LIST_HEAD(&context->vma_private_list); INIT_LIST_HEAD(&context->vma_private_list);
...@@ -1785,9 +1784,6 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev, ...@@ -1785,9 +1784,6 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain)) if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
mlx5_ib_dealloc_transport_domain(dev, context->tdn); mlx5_ib_dealloc_transport_domain(dev, context->tdn);
out_page:
free_page(context->upd_xlt_page);
out_uars: out_uars:
deallocate_uars(dev, context); deallocate_uars(dev, context);
...@@ -1813,7 +1809,6 @@ static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) ...@@ -1813,7 +1809,6 @@ static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain)) if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
mlx5_ib_dealloc_transport_domain(dev, context->tdn); mlx5_ib_dealloc_transport_domain(dev, context->tdn);
free_page(context->upd_xlt_page);
deallocate_uars(dev, context); deallocate_uars(dev, context);
kfree(bfregi->sys_pages); kfree(bfregi->sys_pages);
kfree(bfregi->count); kfree(bfregi->count);
...@@ -5292,13 +5287,32 @@ static struct mlx5_interface mlx5_ib_interface = { ...@@ -5292,13 +5287,32 @@ static struct mlx5_interface mlx5_ib_interface = {
.protocol = MLX5_INTERFACE_PROTOCOL_IB, .protocol = MLX5_INTERFACE_PROTOCOL_IB,
}; };
unsigned long mlx5_ib_get_xlt_emergency_page(void)
{
mutex_lock(&xlt_emergency_page_mutex);
return xlt_emergency_page;
}
void mlx5_ib_put_xlt_emergency_page(void)
{
mutex_unlock(&xlt_emergency_page_mutex);
}
static int __init mlx5_ib_init(void) static int __init mlx5_ib_init(void)
{ {
int err; int err;
xlt_emergency_page = __get_free_page(GFP_KERNEL);
if (!xlt_emergency_page)
return -ENOMEM;
mutex_init(&xlt_emergency_page_mutex);
mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0); mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0);
if (!mlx5_ib_event_wq) if (!mlx5_ib_event_wq) {
free_page(xlt_emergency_page);
return -ENOMEM; return -ENOMEM;
}
mlx5_ib_odp_init(); mlx5_ib_odp_init();
...@@ -5311,6 +5325,8 @@ static void __exit mlx5_ib_cleanup(void) ...@@ -5311,6 +5325,8 @@ static void __exit mlx5_ib_cleanup(void)
{ {
mlx5_unregister_interface(&mlx5_ib_interface); mlx5_unregister_interface(&mlx5_ib_interface);
destroy_workqueue(mlx5_ib_event_wq); destroy_workqueue(mlx5_ib_event_wq);
mutex_destroy(&xlt_emergency_page_mutex);
free_page(xlt_emergency_page);
} }
module_init(mlx5_ib_init); module_init(mlx5_ib_init);
......
...@@ -130,9 +130,6 @@ struct mlx5_ib_ucontext { ...@@ -130,9 +130,6 @@ struct mlx5_ib_ucontext {
/* protect vma_private_list add/del */ /* protect vma_private_list add/del */
struct mutex vma_private_list_mutex; struct mutex vma_private_list_mutex;
unsigned long upd_xlt_page;
/* protect ODP/KSM */
struct mutex upd_xlt_page_mutex;
u64 lib_caps; u64 lib_caps;
}; };
...@@ -1220,4 +1217,7 @@ static inline int get_num_static_uars(struct mlx5_ib_dev *dev, ...@@ -1220,4 +1217,7 @@ static inline int get_num_static_uars(struct mlx5_ib_dev *dev,
return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) * bfregi->num_static_sys_pages; return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) * bfregi->num_static_sys_pages;
} }
unsigned long mlx5_ib_get_xlt_emergency_page(void);
void mlx5_ib_put_xlt_emergency_page(void);
#endif /* MLX5_IB_H */ #endif /* MLX5_IB_H */
...@@ -985,7 +985,6 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, ...@@ -985,7 +985,6 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
{ {
struct mlx5_ib_dev *dev = mr->dev; struct mlx5_ib_dev *dev = mr->dev;
struct device *ddev = dev->ib_dev.dev.parent; struct device *ddev = dev->ib_dev.dev.parent;
struct mlx5_ib_ucontext *uctx = NULL;
int size; int size;
void *xlt; void *xlt;
dma_addr_t dma; dma_addr_t dma;
...@@ -1001,6 +1000,7 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, ...@@ -1001,6 +1000,7 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
size_t pages_to_map = 0; size_t pages_to_map = 0;
size_t pages_iter = 0; size_t pages_iter = 0;
gfp_t gfp; gfp_t gfp;
bool use_emergency_page = false;
/* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes, /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes,
* so we need to align the offset and length accordingly * so we need to align the offset and length accordingly
...@@ -1027,12 +1027,11 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, ...@@ -1027,12 +1027,11 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
} }
if (!xlt) { if (!xlt) {
uctx = to_mucontext(mr->ibmr.pd->uobject->context);
mlx5_ib_warn(dev, "Using XLT emergency buffer\n"); mlx5_ib_warn(dev, "Using XLT emergency buffer\n");
xlt = (void *)mlx5_ib_get_xlt_emergency_page();
size = PAGE_SIZE; size = PAGE_SIZE;
xlt = (void *)uctx->upd_xlt_page;
mutex_lock(&uctx->upd_xlt_page_mutex);
memset(xlt, 0, size); memset(xlt, 0, size);
use_emergency_page = true;
} }
pages_iter = size / desc_size; pages_iter = size / desc_size;
dma = dma_map_single(ddev, xlt, size, DMA_TO_DEVICE); dma = dma_map_single(ddev, xlt, size, DMA_TO_DEVICE);
...@@ -1096,8 +1095,8 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, ...@@ -1096,8 +1095,8 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE); dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);
free_xlt: free_xlt:
if (uctx) if (use_emergency_page)
mutex_unlock(&uctx->upd_xlt_page_mutex); mlx5_ib_put_xlt_emergency_page();
else else
free_pages((unsigned long)xlt, get_order(size)); free_pages((unsigned long)xlt, get_order(size));
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment