diff --git a/drivers/rknpu/rknpu.mod.c b/drivers/rknpu/rknpu.mod.c new file mode 100644 index 0000000000000..041c0daec05da --- /dev/null +++ b/drivers/rknpu/rknpu.mod.c @@ -0,0 +1,31 @@ +#include +#define INCLUDE_VERMAGIC +#include +#include +#include + +BUILD_SALT; + +MODULE_INFO(vermagic, VERMAGIC_STRING); +MODULE_INFO(name, KBUILD_MODNAME); + +__visible struct module __this_module +__section(".gnu.linkonce.this_module") = { + .name = KBUILD_MODNAME, + .init = init_module, +#ifdef CONFIG_MODULE_UNLOAD + .exit = cleanup_module, +#endif + .arch = MODULE_ARCH_INIT, +}; + +MODULE_INFO(intree, "Y"); + +#ifdef CONFIG_RETPOLINE +MODULE_INFO(retpoline, "Y"); +#endif + +MODULE_INFO(depends, ""); + + +MODULE_INFO(srcversion, "3F8407365EF7935915D4CC3"); diff --git a/drivers/rknpu/rknpu_devfreq.c b/drivers/rknpu/rknpu_devfreq.c index 9c5f9b3d256ae..1fe98d567eb54 100644 --- a/drivers/rknpu/rknpu_devfreq.c +++ b/drivers/rknpu/rknpu_devfreq.c @@ -75,6 +75,25 @@ static struct devfreq_governor devfreq_rknpu_ondemand = { .event_handler = devfreq_rknpu_ondemand_handler, }; +static int rk3576_npu_set_read_margin(struct device *dev, + struct rockchip_opp_info *opp_info, + u32 rm) +{ + if (!opp_info->grf || !opp_info->volt_rm_tbl) + return 0; + + if (rm == opp_info->current_rm || rm == UINT_MAX) + return 0; + + LOG_DEV_DEBUG(dev, "set rm to %d\n", rm); + + regmap_write(opp_info->grf, 0x08, 0x001c0000 | (rm << 2)); + regmap_write(opp_info->grf, 0x0c, 0x003c0000 | (rm << 2)); + regmap_write(opp_info->grf, 0x10, 0x001c0000 | (rm << 2)); + + return 0; +} + static int rk3588_npu_get_soc_info(struct device *dev, struct device_node *np, int *bin, int *process) { @@ -213,6 +232,14 @@ static int npu_opp_config_clks(struct device *dev, struct opp_table *opp_table, } #endif +static const struct rockchip_opp_data rk3576_npu_opp_data = { + .set_read_margin = rk3576_npu_set_read_margin, +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE + .config_regulators = npu_opp_config_regulators, + .config_clks = npu_opp_config_clks, +#endif +}; + static const struct rockchip_opp_data rk3588_npu_opp_data = { .get_soc_info = rk3588_npu_get_soc_info, .set_soc_info = rk3588_npu_set_soc_info, @@ -224,6 +251,10 @@ static const struct rockchip_opp_data rk3588_npu_opp_data = { }; static const struct of_device_id rockchip_npu_of_match[] = { + { + .compatible = "rockchip,rk3576", + .data = (void *)&rk3576_npu_opp_data, + }, { .compatible = "rockchip,rk3588", .data = (void *)&rk3588_npu_opp_data, diff --git a/drivers/rknpu/rknpu_drv.c b/drivers/rknpu/rknpu_drv.c index 88783bec35f50..f4f483845b7d8 100644 --- a/drivers/rknpu/rknpu_drv.c +++ b/drivers/rknpu/rknpu_drv.c @@ -43,6 +43,7 @@ #include "rknpu_drv.h" #include "rknpu_gem.h" #include "rknpu_devfreq.h" +#include "rknpu_iommu.h" #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM #include @@ -75,6 +76,11 @@ static const struct rknpu_irqs_data rknpu_irqs[] = { { "npu_irq", rknpu_core0_irq_handler } }; +static const struct rknpu_irqs_data rk3576_npu_irqs[] = { + { "npu0_irq", rknpu_core0_irq_handler }, + { "npu1_irq", rknpu_core1_irq_handler } +}; + static const struct rknpu_irqs_data rk3588_npu_irqs[] = { { "npu0_irq", rknpu_core0_irq_handler }, { "npu1_irq", rknpu_core1_irq_handler }, @@ -197,6 +203,25 @@ static const struct rknpu_config rk3562_rknpu_config = { .amount_core = NULL, }; +static const struct rknpu_config rk3576_rknpu_config = { + .bw_priority_addr = 0x0, + .bw_priority_length = 0x0, + .dma_mask = DMA_BIT_MASK(40), + .pc_data_amount_scale = 2, + .pc_task_number_bits = 16, + .pc_task_number_mask = 0xffff, + .pc_task_status_offset = 0x48, + .pc_dma_ctrl = 1, + .irqs = rk3576_npu_irqs, + .num_irqs = ARRAY_SIZE(rk3576_npu_irqs), + .nbuf_phyaddr = 0x3fe80000, + .nbuf_size = 1024 * 1024, + .max_submit_number = (1 << 16) - 1, + .core_mask = 0x3, + .amount_top = &rknpu_top_amount, + .amount_core = &rknpu_core_amount, +}; + /* driver probe and init */ static const struct of_device_id rknpu_of_match[] = { { @@ -219,6 +244,10 @@ static const struct of_device_id rknpu_of_match[] = { .compatible = "rockchip,rk3562-rknpu", .data = &rk3562_rknpu_config, }, + { + .compatible = "rockchip,rk3576-rknpu", + .data = &rk3576_rknpu_config, + }, {}, }; @@ -375,6 +404,15 @@ static int rknpu_action(struct rknpu_device *rknpu_dev, args->value = 0; ret = 0; break; + case RKNPU_GET_IOMMU_DOMAIN_ID: + args->value = rknpu_dev->iommu_domain_id; + ret = 0; + break; + case RKNPU_SET_IOMMU_DOMAIN_ID: { + ret = rknpu_iommu_switch_domain(rknpu_dev, + *(int32_t *)&args->value); + break; + } default: ret = -EINVAL; break; @@ -464,7 +502,7 @@ static int rknpu_action_ioctl(struct rknpu_device *rknpu_dev, return ret; } -static long rknpu_ioctl(struct file *file, uint32_t cmd, unsigned long arg) +static long rknpu_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { long ret = -EINVAL; struct rknpu_device *rknpu_dev = NULL; @@ -476,22 +514,22 @@ static long rknpu_ioctl(struct file *file, uint32_t cmd, unsigned long arg) rknpu_power_get(rknpu_dev); - switch (cmd) { - case IOCTL_RKNPU_ACTION: + switch (_IOC_NR(cmd)) { + case RKNPU_ACTION: ret = rknpu_action_ioctl(rknpu_dev, arg); break; - case IOCTL_RKNPU_SUBMIT: + case RKNPU_SUBMIT: ret = rknpu_submit_ioctl(rknpu_dev, arg); break; - case IOCTL_RKNPU_MEM_CREATE: - ret = rknpu_mem_create_ioctl(rknpu_dev, arg, file); + case RKNPU_MEM_CREATE: + ret = rknpu_mem_create_ioctl(rknpu_dev, file, cmd, arg); break; case RKNPU_MEM_MAP: break; - case IOCTL_RKNPU_MEM_DESTROY: - ret = rknpu_mem_destroy_ioctl(rknpu_dev, arg, file); + case RKNPU_MEM_DESTROY: + ret = rknpu_mem_destroy_ioctl(rknpu_dev, file, arg); break; - case IOCTL_RKNPU_MEM_SYNC: + case RKNPU_MEM_SYNC: ret = rknpu_mem_sync_ioctl(rknpu_dev, arg); break; default: @@ -1171,7 +1209,11 @@ static int rknpu_probe(struct platform_device *pdev) rknpu_dev->dev = dev; rknpu_dev->iommu_en = rknpu_is_iommu_enable(dev); - if (!rknpu_dev->iommu_en) { + if (rknpu_dev->iommu_en) { + rknpu_dev->iommu_group = iommu_group_get(dev); + if (!rknpu_dev->iommu_group) + return -EINVAL; + } else { /* Initialize reserved memory resources */ ret = of_reserved_mem_device_init(dev); if (!ret) { @@ -1226,6 +1268,7 @@ static int rknpu_probe(struct platform_device *pdev) spin_lock_init(&rknpu_dev->irq_lock); mutex_init(&rknpu_dev->power_lock); mutex_init(&rknpu_dev->reset_lock); + mutex_init(&rknpu_dev->domain_lock); for (i = 0; i < config->num_irqs; i++) { INIT_LIST_HEAD(&rknpu_dev->subcore_datas[i].todo_list); init_waitqueue_head(&rknpu_dev->subcore_datas[i].job_done_wq); @@ -1365,6 +1408,9 @@ static int rknpu_probe(struct platform_device *pdev) rknpu_dev->config->nbuf_size > 0) rknpu_find_nbuf_resource(rknpu_dev); + if (rknpu_dev->iommu_en) + rknpu_iommu_init_domain(rknpu_dev); + rknpu_power_off(rknpu_dev); atomic_set(&rknpu_dev->power_refcount, 0); atomic_set(&rknpu_dev->cmdline_power_refcount, 0); @@ -1401,9 +1447,6 @@ static int rknpu_remove(struct platform_device *pdev) cancel_delayed_work_sync(&rknpu_dev->power_off_work); destroy_workqueue(rknpu_dev->power_off_wq); - if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->sram_mm) - rknpu_mm_destroy(rknpu_dev->sram_mm); - rknpu_debugger_remove(rknpu_dev); rknpu_cancel_timer(rknpu_dev); @@ -1412,6 +1455,14 @@ static int rknpu_remove(struct platform_device *pdev) WARN_ON(!list_empty(&rknpu_dev->subcore_datas[i].todo_list)); } + if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_dev->sram_mm) + rknpu_mm_destroy(rknpu_dev->sram_mm); + + if (rknpu_dev->iommu_en) { + rknpu_iommu_free_domains(rknpu_dev); + iommu_group_put(rknpu_dev->iommu_group); + } + #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM rknpu_drm_remove(rknpu_dev); #endif diff --git a/drivers/rknpu/rknpu_gem.c b/drivers/rknpu/rknpu_gem.c index 60ad44804a5e4..4cabe13a54c0a 100644 --- a/drivers/rknpu/rknpu_gem.c +++ b/drivers/rknpu/rknpu_gem.c @@ -616,7 +616,8 @@ static void rknpu_gem_free_buf_with_cache(struct rknpu_gem_object *rknpu_obj, struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm, unsigned int flags, unsigned long size, - unsigned long sram_size) + unsigned long sram_size, + int iommu_domain_id) { struct rknpu_device *rknpu_dev = drm->dev_private; struct rknpu_gem_object *rknpu_obj = NULL; @@ -630,6 +631,13 @@ struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm, remain_ddr_size = round_up(size, PAGE_SIZE); + rknpu_obj = rknpu_gem_init(drm, remain_ddr_size); + if (IS_ERR(rknpu_obj)) + return rknpu_obj; + + if (!rknpu_iommu_switch_domain(rknpu_dev, iommu_domain_id)) + rknpu_obj->iommu_domain_id = iommu_domain_id; + if (!rknpu_dev->iommu_en && (flags & RKNPU_MEM_NON_CONTIGUOUS)) { /* * when no IOMMU is available, all allocated buffers are @@ -648,10 +656,6 @@ struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm, if (sram_size != 0) sram_size = round_up(sram_size, PAGE_SIZE); - rknpu_obj = rknpu_gem_init(drm, remain_ddr_size); - if (IS_ERR(rknpu_obj)) - return rknpu_obj; - /* set memory type and cache attribute from user side. */ rknpu_obj->flags = flags; @@ -688,15 +692,9 @@ struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm, } else if (IS_ENABLED(CONFIG_NO_GKI) && (flags & RKNPU_MEM_TRY_ALLOC_NBUF) && rknpu_dev->nbuf_size > 0) { - size_t nbuf_size = 0; - - rknpu_obj = rknpu_gem_init(drm, remain_ddr_size); - if (IS_ERR(rknpu_obj)) - return rknpu_obj; - - nbuf_size = remain_ddr_size <= rknpu_dev->nbuf_size ? - remain_ddr_size : - rknpu_dev->nbuf_size; + size_t nbuf_size = remain_ddr_size <= rknpu_dev->nbuf_size ? + remain_ddr_size : + rknpu_dev->nbuf_size; /* set memory type and cache attribute from user side. */ rknpu_obj->flags = flags; @@ -713,10 +711,6 @@ struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm, } if (remain_ddr_size > 0) { - rknpu_obj = rknpu_gem_init(drm, remain_ddr_size); - if (IS_ERR(rknpu_obj)) - return rknpu_obj; - /* set memory type and cache attribute from user side. */ rknpu_obj->flags = flags; @@ -725,13 +719,12 @@ struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm, goto gem_release; } - if (rknpu_obj) - LOG_DEBUG( - "created dma addr: %pad, cookie: %p, ddr size: %lu, sram size: %lu, nbuf size: %lu, attrs: %#lx, flags: %#x\n", - &rknpu_obj->dma_addr, rknpu_obj->cookie, - rknpu_obj->size, rknpu_obj->sram_size, - rknpu_obj->nbuf_size, rknpu_obj->dma_attrs, - rknpu_obj->flags); + LOG_DEBUG( + "created dma addr: %pad, cookie: %p, ddr size: %lu, sram size: %lu, nbuf size: %lu, attrs: %#lx, flags: %#x, iommu domain id: %d\n", + &rknpu_obj->dma_addr, rknpu_obj->cookie, rknpu_obj->size, + rknpu_obj->sram_size, rknpu_obj->nbuf_size, + rknpu_obj->dma_attrs, rknpu_obj->flags, + rknpu_obj->iommu_domain_id); return rknpu_obj; @@ -749,12 +742,15 @@ struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *drm, void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj) { struct drm_gem_object *obj = &rknpu_obj->base; + struct rknpu_device *rknpu_dev = obj->dev->dev_private; LOG_DEBUG( "destroy dma addr: %pad, cookie: %p, size: %lu, attrs: %#lx, flags: %#x, handle count: %d\n", &rknpu_obj->dma_addr, rknpu_obj->cookie, rknpu_obj->size, rknpu_obj->dma_attrs, rknpu_obj->flags, obj->handle_count); + rknpu_iommu_switch_domain(rknpu_dev, rknpu_obj->iommu_domain_id); + /* * do not release memory region from exporter. * @@ -767,8 +763,6 @@ void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj) } else { if (IS_ENABLED(CONFIG_ROCKCHIP_RKNPU_SRAM) && rknpu_obj->sram_size > 0) { - struct rknpu_device *rknpu_dev = obj->dev->dev_private; - if (rknpu_obj->sram_obj != NULL) rknpu_mm_free(rknpu_dev->sram_mm, rknpu_obj->sram_obj); @@ -786,7 +780,7 @@ void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj) rknpu_gem_release(rknpu_obj); } -int rknpu_gem_create_ioctl(struct drm_device *dev, void *data, +int rknpu_gem_create_ioctl(struct drm_device *drm, void *data, struct drm_file *file_priv) { struct rknpu_mem_create *args = data; @@ -795,8 +789,9 @@ int rknpu_gem_create_ioctl(struct drm_device *dev, void *data, rknpu_obj = rknpu_gem_object_find(file_priv, args->handle); if (!rknpu_obj) { - rknpu_obj = rknpu_gem_object_create( - dev, args->flags, args->size, args->sram_size); + rknpu_obj = rknpu_gem_object_create(drm, args->flags, + args->size, args->sram_size, + args->iommu_domain_id); if (IS_ERR(rknpu_obj)) return PTR_ERR(rknpu_obj); @@ -832,9 +827,10 @@ int rknpu_gem_map_ioctl(struct drm_device *dev, void *data, #endif } -int rknpu_gem_destroy_ioctl(struct drm_device *dev, void *data, +int rknpu_gem_destroy_ioctl(struct drm_device *drm, void *data, struct drm_file *file_priv) { + struct rknpu_device *rknpu_dev = drm->dev_private; struct rknpu_gem_object *rknpu_obj = NULL; struct rknpu_mem_destroy *args = data; @@ -842,6 +838,8 @@ int rknpu_gem_destroy_ioctl(struct drm_device *dev, void *data, if (!rknpu_obj) return -EINVAL; + rknpu_iommu_switch_domain(rknpu_dev, rknpu_obj->iommu_domain_id); + // rknpu_gem_object_put(&rknpu_obj->base); return rknpu_gem_handle_destroy(file_priv, args->handle); @@ -1045,7 +1043,7 @@ int rknpu_gem_dumb_create(struct drm_file *file_priv, struct drm_device *drm, else flags = RKNPU_MEM_CONTIGUOUS | RKNPU_MEM_WRITE_COMBINE; - rknpu_obj = rknpu_gem_object_create(drm, flags, args->size, 0); + rknpu_obj = rknpu_gem_object_create(drm, flags, args->size, 0, 0); if (IS_ERR(rknpu_obj)) { LOG_DEV_ERROR(drm->dev, "gem object allocate failed.\n"); return PTR_ERR(rknpu_obj); diff --git a/drivers/rknpu/rknpu_iommu.c b/drivers/rknpu/rknpu_iommu.c index 01620d9c30560..53bca78953ac4 100644 --- a/drivers/rknpu/rknpu_iommu.c +++ b/drivers/rknpu/rknpu_iommu.c @@ -59,3 +59,196 @@ void rknpu_iommu_dma_free_iova(struct rknpu_iommu_dma_cookie *cookie, free_iova_fast(iovad, iova_pfn(iovad, iova), size >> iova_shift(iovad)); } + +#if defined(CONFIG_IOMMU_API) && defined(CONFIG_NO_GKI) + +#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE +struct iommu_group { + struct kobject kobj; + struct kobject *devices_kobj; + struct list_head devices; +#ifdef __ANDROID_COMMON_KERNEL__ + struct xarray pasid_array; +#endif + struct mutex mutex; + void *iommu_data; + void (*iommu_data_release)(void *iommu_data); + char *name; + int id; + struct iommu_domain *default_domain; + struct iommu_domain *blocking_domain; + struct iommu_domain *domain; + struct list_head entry; + unsigned int owner_cnt; + void *owner; +}; +#else +struct iommu_group { + struct kobject kobj; + struct kobject *devices_kobj; + struct list_head devices; + struct mutex mutex; + struct blocking_notifier_head notifier; + void *iommu_data; + void (*iommu_data_release)(void *iommu_data); + char *name; + int id; + struct iommu_domain *default_domain; + struct iommu_domain *domain; + struct list_head entry; +}; +#endif + +int rknpu_iommu_init_domain(struct rknpu_device *rknpu_dev) +{ + // init domain 0 + if (!rknpu_dev->iommu_domains[0]) { + rknpu_dev->iommu_domain_id = 0; + rknpu_dev->iommu_domains[rknpu_dev->iommu_domain_id] = + iommu_get_domain_for_dev(rknpu_dev->dev); + rknpu_dev->iommu_domain_num = 1; + } + return 0; +} + +int rknpu_iommu_switch_domain(struct rknpu_device *rknpu_dev, int domain_id) +{ + struct iommu_domain *src_domain = NULL; + struct iommu_domain *dst_domain = NULL; + struct bus_type *bus = NULL; + int src_domain_id = 0; + int ret = -EINVAL; + + if (!rknpu_dev->iommu_en) + return -EINVAL; + + if (domain_id < 0 || domain_id > (RKNPU_MAX_IOMMU_DOMAIN_NUM - 1)) { + LOG_DEV_ERROR( + rknpu_dev->dev, + "invalid iommu domain id: %d, reuse domain id: %d\n", + domain_id, rknpu_dev->iommu_domain_id); + return -EINVAL; + } + + bus = rknpu_dev->dev->bus; + if (!bus) + return -EFAULT; + + mutex_lock(&rknpu_dev->domain_lock); + + src_domain_id = rknpu_dev->iommu_domain_id; + if (domain_id == src_domain_id) { + mutex_unlock(&rknpu_dev->domain_lock); + return 0; + } + + src_domain = iommu_get_domain_for_dev(rknpu_dev->dev); + if (src_domain != rknpu_dev->iommu_domains[src_domain_id]) { + LOG_DEV_ERROR( + rknpu_dev->dev, + "mismatch domain get from iommu_get_domain_for_dev\n"); + mutex_unlock(&rknpu_dev->domain_lock); + return -EINVAL; + } + + dst_domain = rknpu_dev->iommu_domains[domain_id]; + if (dst_domain != NULL) { + iommu_detach_device(src_domain, rknpu_dev->dev); + ret = iommu_attach_device(dst_domain, rknpu_dev->dev); + if (ret) { + LOG_DEV_ERROR( + rknpu_dev->dev, + "failed to attach dst iommu domain, id: %d, ret: %d\n", + domain_id, ret); + if (iommu_attach_device(src_domain, rknpu_dev->dev)) { + LOG_DEV_ERROR( + rknpu_dev->dev, + "failed to reattach src iommu domain, id: %d\n", + src_domain_id); + } + mutex_unlock(&rknpu_dev->domain_lock); + return ret; + } + rknpu_dev->iommu_domain_id = domain_id; + } else { + uint64_t dma_limit = 1ULL << 32; + + dst_domain = iommu_domain_alloc(bus); + if (!dst_domain) { + LOG_DEV_ERROR(rknpu_dev->dev, + "failed to allocate iommu domain\n"); + mutex_unlock(&rknpu_dev->domain_lock); + return -EIO; + } + // init domain iova_cookie + iommu_get_dma_cookie(dst_domain); + + iommu_detach_device(src_domain, rknpu_dev->dev); + ret = iommu_attach_device(dst_domain, rknpu_dev->dev); + if (ret) { + LOG_DEV_ERROR( + rknpu_dev->dev, + "failed to attach iommu domain, id: %d, ret: %d\n", + domain_id, ret); + iommu_domain_free(dst_domain); + mutex_unlock(&rknpu_dev->domain_lock); + return ret; + } + + // set domain type to dma domain + dst_domain->type |= __IOMMU_DOMAIN_DMA_API; + // iommu dma init domain + iommu_setup_dma_ops(rknpu_dev->dev, 0, dma_limit); + + rknpu_dev->iommu_domain_id = domain_id; + rknpu_dev->iommu_domains[domain_id] = dst_domain; + rknpu_dev->iommu_domain_num++; + } + + // reset default iommu domain + rknpu_dev->iommu_group->default_domain = dst_domain; + + mutex_unlock(&rknpu_dev->domain_lock); + + LOG_INFO("switch iommu domain from %d to %d\n", src_domain_id, + domain_id); + + return ret; +} + +void rknpu_iommu_free_domains(struct rknpu_device *rknpu_dev) +{ + int i = 0; + + rknpu_iommu_switch_domain(rknpu_dev, 0); + + for (i = 1; i < RKNPU_MAX_IOMMU_DOMAIN_NUM; i++) { + struct iommu_domain *domain = rknpu_dev->iommu_domains[i]; + + if (domain == NULL) + continue; + + iommu_detach_device(domain, rknpu_dev->dev); + iommu_domain_free(domain); + + rknpu_dev->iommu_domains[i] = NULL; + } +} + +#else + +int rknpu_iommu_init_domain(struct rknpu_device *rknpu_dev) +{ + return 0; +} + +int rknpu_iommu_switch_domain(struct rknpu_device *rknpu_dev, int domain_id) +{ + return 0; +} + +void rknpu_iommu_free_domains(struct rknpu_device *rknpu_dev) +{ +} + +#endif diff --git a/drivers/rknpu/rknpu_job.c b/drivers/rknpu/rknpu_job.c index e13488dfc6b36..312d9c73df75a 100644 --- a/drivers/rknpu/rknpu_job.c +++ b/drivers/rknpu/rknpu_job.c @@ -14,8 +14,9 @@ #include "rknpu_reset.h" #include "rknpu_gem.h" #include "rknpu_fence.h" -#include "rknpu_job.h" #include "rknpu_mem.h" +#include "rknpu_iommu.h" +#include "rknpu_job.h" #define _REG_READ(base, offset) readl(base + (offset)) #define _REG_WRITE(base, value, offset) writel(value, base + (offset)) @@ -144,6 +145,7 @@ static inline struct rknpu_job *rknpu_job_alloc(struct rknpu_device *rknpu_dev, ((args->core_mask & RKNPU_CORE2_MASK) >> 2); atomic_set(&job->run_count, job->use_core_num); atomic_set(&job->interrupt_count, job->use_core_num); + job->iommu_domain_id = args->iommu_domain_id; for (i = 0; i < rknpu_dev->config->num_irqs; i++) atomic_set(&job->submit_count[i], 0); #ifdef CONFIG_ROCKCHIP_RKNPU_DRM_GEM @@ -208,8 +210,9 @@ static inline int rknpu_job_wait(struct rknpu_job *job) (elapse_time_us < args->timeout * 1000); spin_unlock_irqrestore(&rknpu_dev->irq_lock, flags); LOG_ERROR( - "job: %p, wait_count: %d, continue wait: %d, commit elapse time: %lldus, wait time: %lldus, timeout: %uus\n", - job, wait_count, continue_wait, + "job: %p, iommu domain id: %d, wait_count: %d, continue wait: %d, commit elapse time: %lldus, wait time: %lldus, timeout: %uus\n", + job, job->iommu_domain_id, wait_count, + continue_wait, (job->hw_commit_time == 0 ? 0 : elapse_time_us), ktime_us_delta(ktime_get(), job->timestamp), args->timeout * 1000); @@ -840,6 +843,8 @@ int rknpu_submit_ioctl(struct drm_device *dev, void *data, struct rknpu_device *rknpu_dev = dev_get_drvdata(dev->dev); struct rknpu_submit *args = data; + rknpu_iommu_switch_domain(rknpu_dev, args->iommu_domain_id); + return rknpu_submit(rknpu_dev, args); } #endif diff --git a/drivers/rknpu/rknpu_mem.c b/drivers/rknpu/rknpu_mem.c index 858c21f484f5e..31aede9515959 100644 --- a/drivers/rknpu/rknpu_mem.c +++ b/drivers/rknpu/rknpu_mem.c @@ -17,8 +17,8 @@ #ifdef CONFIG_ROCKCHIP_RKNPU_DMA_HEAP -int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data, - struct file *file) +int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, struct file *file, + unsigned int cmd, unsigned long data) { struct rknpu_mem_create args; int ret = -EINVAL; @@ -33,14 +33,20 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data, struct rknpu_session *session = NULL; int i, fd; unsigned int length, page_count; + unsigned int in_size = _IOC_SIZE(cmd); + unsigned int k_size = sizeof(struct rknpu_mem_create); + char *k_data = (char *)&args; if (unlikely(copy_from_user(&args, (struct rknpu_mem_create *)data, - sizeof(struct rknpu_mem_create)))) { + in_size))) { LOG_ERROR("%s: copy_from_user failed\n", __func__); ret = -EFAULT; return ret; } + if (k_size > in_size) + memset(k_data + in_size, 0, k_size - in_size); + if (args.flags & RKNPU_MEM_NON_CONTIGUOUS) { LOG_ERROR("%s: malloc iommu memory unsupported in current!\n", __func__); @@ -147,7 +153,7 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data, (__u64)rknpu_obj->dma_addr); if (unlikely(copy_to_user((struct rknpu_mem_create *)data, &args, - sizeof(struct rknpu_mem_create)))) { + in_size))) { LOG_ERROR("%s: copy_to_user failed\n", __func__); ret = -EFAULT; goto err_unmap_kv_addr; @@ -194,8 +200,8 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data, return ret; } -int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, unsigned long data, - struct file *file) +int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, struct file *file, + unsigned long data) { struct rknpu_mem_object *rknpu_obj, *entry, *q; struct rknpu_session *session = NULL;