Commit f3a39818 authored by Andrew Lewycky's avatar Andrew Lewycky Committed by Oded Gabbay

drm/amdkfd: Add the events module

This patch adds the events module (kfd_events.c) and the interrupt
handle module for Kaveri (cik_event_interrupt.c).

The patch updates the interrupt_is_wanted(), so that it now calls the
interrupt isr function specific for the device that received the
interrupt. That function(implemented in cik_event_interrupt.c)
returns whether this interrupt is of interest to us or not.

The patch also updates the interrupt_wq(), so that it now calls the
device's specific wq function, which checks the interrupt source
and tries to signal relevant events.

v2:

Increase limit of signal events to 4096 per process
Remove bitfields from struct cik_ih_ring_entry
Rename radeon_kfd_event_mmap to kfd_event_mmap
Add debug prints to allocate_free_slot and allocate_signal_page
Make allocate_event_notification_slot return a correct value
Add warning prints to create_signal_event
Remove error print from IOCTL path
Reformatted debug prints in kfd_event_mmap
Map correct size (as received from mmap) in kfd_event_mmap

v3:

Reduce limit of signal events back to 256 per process
Fix allocation of kernel memory for signal events
Signed-off-by: default avatarAndrew Lewycky <Andrew.Lewycky@amd.com>
Signed-off-by: default avatarOded Gabbay <oded.gabbay@gmail.com>
parent 29a5d3eb
...@@ -12,6 +12,6 @@ amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \ ...@@ -12,6 +12,6 @@ amdkfd-y := kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \
kfd_kernel_queue_vi.o kfd_packet_manager.o \ kfd_kernel_queue_vi.o kfd_packet_manager.o \
kfd_process_queue_manager.o kfd_device_queue_manager.o \ kfd_process_queue_manager.o kfd_device_queue_manager.o \
kfd_device_queue_manager_cik.o kfd_device_queue_manager_vi.o \ kfd_device_queue_manager_cik.o kfd_device_queue_manager_vi.o \
kfd_interrupt.o kfd_interrupt.o kfd_events.o cik_event_interrupt.o
obj-$(CONFIG_HSA_AMD) += amdkfd.o obj-$(CONFIG_HSA_AMD) += amdkfd.o
/*
* Copyright 2014 Advanced Micro Devices, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#include "kfd_priv.h"
#include "kfd_events.h"
#include "cik_int.h"
static bool cik_event_interrupt_isr(struct kfd_dev *dev,
const uint32_t *ih_ring_entry)
{
unsigned int pasid;
const struct cik_ih_ring_entry *ihre =
(const struct cik_ih_ring_entry *)ih_ring_entry;
pasid = (ihre->ring_id & 0xffff0000) >> 16;
/* Do not process in ISR, just request it to be forwarded to WQ. */
return (pasid != 0) &&
(ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE ||
ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG);
}
static void cik_event_interrupt_wq(struct kfd_dev *dev,
const uint32_t *ih_ring_entry)
{
unsigned int pasid;
const struct cik_ih_ring_entry *ihre =
(const struct cik_ih_ring_entry *)ih_ring_entry;
pasid = (ihre->ring_id & 0xffff0000) >> 16;
if (pasid == 0)
return;
if (ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE)
kfd_signal_event_interrupt(pasid, 0, 0);
else if (ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG)
kfd_signal_event_interrupt(pasid, ihre->data & 0xFF, 8);
}
const struct kfd_event_interrupt_class event_interrupt_class_cik = {
.interrupt_isr = cik_event_interrupt_isr,
.interrupt_wq = cik_event_interrupt_wq,
};
/*
* Copyright 2014 Advanced Micro Devices, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef HSA_RADEON_CIK_INT_H_INCLUDED
#define HSA_RADEON_CIK_INT_H_INCLUDED
#include <linux/types.h>
struct cik_ih_ring_entry {
uint32_t source_id;
uint32_t data;
uint32_t ring_id;
uint32_t reserved;
};
#define CIK_INTSRC_DEQUEUE_COMPLETE 0xC6
#define CIK_INTSRC_CP_END_OF_PIPE 0xB5
#define CIK_INTSRC_SQ_INTERRUPT_MSG 0xEF
#endif
...@@ -289,8 +289,10 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, ...@@ -289,8 +289,10 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
args->queue_id = queue_id; args->queue_id = queue_id;
/* Return gpu_id as doorbell offset for mmap usage */ /* Return gpu_id as doorbell offset for mmap usage */
args->doorbell_offset = args->gpu_id << PAGE_SHIFT; args->doorbell_offset = (KFD_MMAP_DOORBELL_MASK | args->gpu_id);
args->doorbell_offset <<= PAGE_SHIFT;
mutex_unlock(&p->mutex); mutex_unlock(&p->mutex);
...@@ -684,5 +686,15 @@ static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) ...@@ -684,5 +686,15 @@ static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
if (IS_ERR(process)) if (IS_ERR(process))
return PTR_ERR(process); return PTR_ERR(process);
return kfd_doorbell_mmap(process, vma); if ((vma->vm_pgoff & KFD_MMAP_DOORBELL_MASK) ==
KFD_MMAP_DOORBELL_MASK) {
vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_DOORBELL_MASK;
return kfd_doorbell_mmap(process, vma);
} else if ((vma->vm_pgoff & KFD_MMAP_EVENTS_MASK) ==
KFD_MMAP_EVENTS_MASK) {
vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_EVENTS_MASK;
return kfd_event_mmap(process, vma);
}
return -EFAULT;
} }
...@@ -34,6 +34,7 @@ static const struct kfd_device_info kaveri_device_info = { ...@@ -34,6 +34,7 @@ static const struct kfd_device_info kaveri_device_info = {
.asic_family = CHIP_KAVERI, .asic_family = CHIP_KAVERI,
.max_pasid_bits = 16, .max_pasid_bits = 16,
.ih_ring_entry_size = 4 * sizeof(uint32_t), .ih_ring_entry_size = 4 * sizeof(uint32_t),
.event_interrupt_class = &event_interrupt_class_cik,
.mqd_size_aligned = MQD_SIZE_ALIGNED .mqd_size_aligned = MQD_SIZE_ALIGNED
}; };
......
This diff is collapsed.
/*
* Copyright 2014 Advanced Micro Devices, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef KFD_EVENTS_H_INCLUDED
#define KFD_EVENTS_H_INCLUDED
#include <linux/kernel.h>
#include <linux/hashtable.h>
#include <linux/types.h>
#include <linux/list.h>
#include "kfd_priv.h"
#define KFD_EVENT_ID_NONSIGNAL_MASK 0x80000000U
#define KFD_FIRST_NONSIGNAL_EVENT_ID KFD_EVENT_ID_NONSIGNAL_MASK
#define KFD_LAST_NONSIGNAL_EVENT_ID UINT_MAX
/*
* Written into kfd_signal_slot_t to indicate that the event is not signaled.
* Since the event protocol may need to write the event ID into memory, this
* must not be a valid event ID.
* For the sake of easy memset-ing, this must be a byte pattern.
*/
#define UNSIGNALED_EVENT_SLOT ((uint64_t)-1)
struct kfd_event_waiter;
struct signal_page;
struct kfd_event {
/* All events in process, rooted at kfd_process.events. */
struct hlist_node events;
u32 event_id;
bool signaled;
bool auto_reset;
int type;
struct list_head waiters; /* List of kfd_event_waiter by waiters. */
/* Only for signal events. */
struct signal_page *signal_page;
unsigned int signal_slot_index;
uint64_t __user *user_signal_address;
};
#define KFD_EVENT_TIMEOUT_IMMEDIATE 0
#define KFD_EVENT_TIMEOUT_INFINITE 0xFFFFFFFFu
/* Matching HSA_EVENTTYPE */
#define KFD_EVENT_TYPE_SIGNAL 0
#define KFD_EVENT_TYPE_DEBUG 5
extern void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
uint32_t valid_id_bits);
#endif
...@@ -172,10 +172,17 @@ static void interrupt_wq(struct work_struct *work) ...@@ -172,10 +172,17 @@ static void interrupt_wq(struct work_struct *work)
sizeof(uint32_t))]; sizeof(uint32_t))];
while (dequeue_ih_ring_entry(dev, ih_ring_entry)) while (dequeue_ih_ring_entry(dev, ih_ring_entry))
; dev->device_info->event_interrupt_class->interrupt_wq(dev,
ih_ring_entry);
} }
bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry) bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry)
{ {
return false; /* integer and bitwise OR so there is no boolean short-circuiting */
unsigned wanted = 0;
wanted |= dev->device_info->event_interrupt_class->interrupt_isr(dev,
ih_ring_entry);
return wanted != 0;
} }
...@@ -35,6 +35,9 @@ ...@@ -35,6 +35,9 @@
#define KFD_SYSFS_FILE_MODE 0444 #define KFD_SYSFS_FILE_MODE 0444
#define KFD_MMAP_DOORBELL_MASK 0x8000000000000
#define KFD_MMAP_EVENTS_MASK 0x4000000000000
/* /*
* When working with cp scheduler we should assign the HIQ manually or via * When working with cp scheduler we should assign the HIQ manually or via
* the radeon driver to a fixed hqd slot, here are the fixed HIQ hqd slot * the radeon driver to a fixed hqd slot, here are the fixed HIQ hqd slot
...@@ -108,8 +111,16 @@ enum asic_family_type { ...@@ -108,8 +111,16 @@ enum asic_family_type {
CHIP_CARRIZO CHIP_CARRIZO
}; };
struct kfd_event_interrupt_class {
bool (*interrupt_isr)(struct kfd_dev *dev,
const uint32_t *ih_ring_entry);
void (*interrupt_wq)(struct kfd_dev *dev,
const uint32_t *ih_ring_entry);
};
struct kfd_device_info { struct kfd_device_info {
unsigned int asic_family; unsigned int asic_family;
const struct kfd_event_interrupt_class *event_interrupt_class;
unsigned int max_pasid_bits; unsigned int max_pasid_bits;
size_t ih_ring_entry_size; size_t ih_ring_entry_size;
uint8_t num_of_watch_points; uint8_t num_of_watch_points;
...@@ -490,6 +501,15 @@ struct kfd_process { ...@@ -490,6 +501,15 @@ struct kfd_process {
/*Is the user space process 32 bit?*/ /*Is the user space process 32 bit?*/
bool is_32bit_user_mode; bool is_32bit_user_mode;
/* Event-related data */
struct mutex event_mutex;
/* All events in process hashed by ID, linked on kfd_event.events. */
DECLARE_HASHTABLE(events, 4);
struct list_head signal_event_pages; /* struct slot_page_header.
event_pages */
u32 next_nonsignal_event_id;
size_t signal_event_count;
}; };
/** /**
...@@ -514,6 +534,7 @@ void kfd_process_create_wq(void); ...@@ -514,6 +534,7 @@ void kfd_process_create_wq(void);
void kfd_process_destroy_wq(void); void kfd_process_destroy_wq(void);
struct kfd_process *kfd_create_process(const struct task_struct *); struct kfd_process *kfd_create_process(const struct task_struct *);
struct kfd_process *kfd_get_process(const struct task_struct *); struct kfd_process *kfd_get_process(const struct task_struct *);
struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid);
struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
struct kfd_process *p); struct kfd_process *p);
...@@ -659,4 +680,30 @@ uint64_t kfd_get_number_elems(struct kfd_dev *kfd); ...@@ -659,4 +680,30 @@ uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev, phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
struct kfd_process *process); struct kfd_process *process);
/* Events */
extern const struct kfd_event_interrupt_class event_interrupt_class_cik;
enum kfd_event_wait_result {
KFD_WAIT_COMPLETE,
KFD_WAIT_TIMEOUT,
KFD_WAIT_ERROR
};
void kfd_event_init_process(struct kfd_process *p);
void kfd_event_free_process(struct kfd_process *p);
int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma);
int kfd_wait_on_events(struct kfd_process *p,
uint32_t num_events, const uint32_t __user *event_ids,
bool all, uint32_t user_timeout_ms,
enum kfd_event_wait_result *wait_result);
void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
uint32_t valid_id_bits);
int kfd_set_event(struct kfd_process *p, uint32_t event_id);
int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
int kfd_event_create(struct file *devkfd, struct kfd_process *p,
uint32_t event_type, bool auto_reset, uint32_t node_id,
uint32_t *event_id, uint32_t *event_trigger_data,
uint64_t *event_page_offset, uint32_t *event_slot_index);
int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
#endif #endif
...@@ -178,6 +178,8 @@ static void kfd_process_wq_release(struct work_struct *work) ...@@ -178,6 +178,8 @@ static void kfd_process_wq_release(struct work_struct *work)
kfree(pdd); kfree(pdd);
} }
kfd_event_free_process(p);
kfd_pasid_free(p->pasid); kfd_pasid_free(p->pasid);
mutex_unlock(&p->mutex); mutex_unlock(&p->mutex);
...@@ -288,6 +290,8 @@ static struct kfd_process *create_process(const struct task_struct *thread) ...@@ -288,6 +290,8 @@ static struct kfd_process *create_process(const struct task_struct *thread)
INIT_LIST_HEAD(&process->per_device_data); INIT_LIST_HEAD(&process->per_device_data);
kfd_event_init_process(process);
err = pqm_init(&process->pqm, process); err = pqm_init(&process->pqm, process);
if (err != 0) if (err != 0)
goto err_process_pqm_init; goto err_process_pqm_init;
...@@ -430,3 +434,23 @@ bool kfd_has_process_device_data(struct kfd_process *p) ...@@ -430,3 +434,23 @@ bool kfd_has_process_device_data(struct kfd_process *p)
{ {
return !(list_empty(&p->per_device_data)); return !(list_empty(&p->per_device_data));
} }
/* This returns with process->mutex locked. */
struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid)
{
struct kfd_process *p;
unsigned int temp;
int idx = srcu_read_lock(&kfd_processes_srcu);
hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
if (p->pasid == pasid) {
mutex_lock(&p->mutex);
break;
}
}
srcu_read_unlock(&kfd_processes_srcu, idx);
return p;
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment