diff options
Diffstat (limited to 'drivers/gpu/drm/panfrost')
| -rw-r--r-- | drivers/gpu/drm/panfrost/Makefile | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/TODO | 15 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_devfreq.c | 22 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_devfreq.h | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_device.c | 28 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_device.h | 31 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_drv.c | 196 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_gem.c | 142 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_gem.h | 23 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c | 110 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_gpu.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_job.c | 62 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_mmu.c | 442 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_mmu.h | 9 |
14 files changed, 847 insertions, 237 deletions
diff --git a/drivers/gpu/drm/panfrost/Makefile b/drivers/gpu/drm/panfrost/Makefile index ecf0864cb515..b71935862417 100644 --- a/drivers/gpu/drm/panfrost/Makefile +++ b/drivers/gpu/drm/panfrost/Makefile @@ -5,6 +5,7 @@ panfrost-y := \ panfrost_device.o \ panfrost_devfreq.o \ panfrost_gem.o \ + panfrost_gem_shrinker.o \ panfrost_gpu.o \ panfrost_job.o \ panfrost_mmu.o \ diff --git a/drivers/gpu/drm/panfrost/TODO b/drivers/gpu/drm/panfrost/TODO index c2e44add37d8..536a0d4f8d29 100644 --- a/drivers/gpu/drm/panfrost/TODO +++ b/drivers/gpu/drm/panfrost/TODO @@ -6,22 +6,7 @@ - Bifrost specific feature and issue handling - Coherent DMA support -- Support for 2MB pages. The io-pgtable code already supports this. Finishing - support involves either copying or adapting the iommu API to handle passing - aligned addresses and sizes to the io-pgtable code. - -- Per FD address space support. The h/w supports multiple addresses spaces. - The hard part is handling when more address spaces are needed than what - the h/w provides. - -- Support pinning pages on demand (GPU page faults). - - Support userspace controlled GPU virtual addresses. Needed for Vulkan. (Tomeu) -- Support for madvise and a shrinker. - - Compute job support. So called 'compute only' jobs need to be plumbed up to userspace. - -- Performance counter support. (Boris) - diff --git a/drivers/gpu/drm/panfrost/panfrost_devfreq.c b/drivers/gpu/drm/panfrost/panfrost_devfreq.c index db798532b0b6..a1f5fa6a742a 100644 --- a/drivers/gpu/drm/panfrost/panfrost_devfreq.c +++ b/drivers/gpu/drm/panfrost/panfrost_devfreq.c @@ -39,7 +39,7 @@ static int panfrost_devfreq_target(struct device *dev, unsigned long *freq, * If frequency scaling from low to high, adjust voltage first. * If frequency scaling from high to low, adjust frequency first. */ - if (old_clk_rate < target_rate) { + if (old_clk_rate < target_rate && pfdev->regulator) { err = regulator_set_voltage(pfdev->regulator, target_volt, target_volt); if (err) { @@ -53,12 +53,14 @@ static int panfrost_devfreq_target(struct device *dev, unsigned long *freq, if (err) { dev_err(dev, "Cannot set frequency %lu (%d)\n", target_rate, err); - regulator_set_voltage(pfdev->regulator, pfdev->devfreq.cur_volt, - pfdev->devfreq.cur_volt); + if (pfdev->regulator) + regulator_set_voltage(pfdev->regulator, + pfdev->devfreq.cur_volt, + pfdev->devfreq.cur_volt); return err; } - if (old_clk_rate > target_rate) { + if (old_clk_rate > target_rate && pfdev->regulator) { err = regulator_set_voltage(pfdev->regulator, target_volt, target_volt); if (err) @@ -136,9 +138,6 @@ int panfrost_devfreq_init(struct panfrost_device *pfdev) int ret; struct dev_pm_opp *opp; - if (!pfdev->regulator) - return 0; - ret = dev_pm_opp_of_add_table(&pfdev->pdev->dev); if (ret == -ENODEV) /* Optional, continue without devfreq */ return 0; @@ -157,17 +156,24 @@ int panfrost_devfreq_init(struct panfrost_device *pfdev) dev_pm_opp_put(opp); pfdev->devfreq.devfreq = devm_devfreq_add_device(&pfdev->pdev->dev, - &panfrost_devfreq_profile, "simple_ondemand", NULL); + &panfrost_devfreq_profile, DEVFREQ_GOV_SIMPLE_ONDEMAND, + NULL); if (IS_ERR(pfdev->devfreq.devfreq)) { DRM_DEV_ERROR(&pfdev->pdev->dev, "Couldn't initialize GPU devfreq\n"); ret = PTR_ERR(pfdev->devfreq.devfreq); pfdev->devfreq.devfreq = NULL; + dev_pm_opp_of_remove_table(&pfdev->pdev->dev); return ret; } return 0; } +void panfrost_devfreq_fini(struct panfrost_device *pfdev) +{ + dev_pm_opp_of_remove_table(&pfdev->pdev->dev); +} + void panfrost_devfreq_resume(struct panfrost_device *pfdev) { int i; diff --git a/drivers/gpu/drm/panfrost/panfrost_devfreq.h b/drivers/gpu/drm/panfrost/panfrost_devfreq.h index eb999531ed90..e3bc63e82843 100644 --- a/drivers/gpu/drm/panfrost/panfrost_devfreq.h +++ b/drivers/gpu/drm/panfrost/panfrost_devfreq.h @@ -5,6 +5,7 @@ #define __PANFROST_DEVFREQ_H__ int panfrost_devfreq_init(struct panfrost_device *pfdev); +void panfrost_devfreq_fini(struct panfrost_device *pfdev); void panfrost_devfreq_resume(struct panfrost_device *pfdev); void panfrost_devfreq_suspend(struct panfrost_device *pfdev); diff --git a/drivers/gpu/drm/panfrost/panfrost_device.c b/drivers/gpu/drm/panfrost/panfrost_device.c index 8a111d7c0200..46b0b02e4289 100644 --- a/drivers/gpu/drm/panfrost/panfrost_device.c +++ b/drivers/gpu/drm/panfrost/panfrost_device.c @@ -5,7 +5,6 @@ #include <linux/clk.h> #include <linux/reset.h> #include <linux/platform_device.h> -#include <linux/pm_runtime.h> #include <linux/regulator/consumer.h> #include "panfrost_device.h" @@ -123,8 +122,9 @@ int panfrost_device_init(struct panfrost_device *pfdev) mutex_init(&pfdev->sched_lock); mutex_init(&pfdev->reset_lock); INIT_LIST_HEAD(&pfdev->scheduled_jobs); + INIT_LIST_HEAD(&pfdev->as_lru_list); - spin_lock_init(&pfdev->hwaccess_lock); + spin_lock_init(&pfdev->as_lock); err = panfrost_clk_init(pfdev); if (err) { @@ -164,14 +164,6 @@ int panfrost_device_init(struct panfrost_device *pfdev) if (err) goto err_out4; - /* runtime PM will wake us up later */ - panfrost_gpu_power_off(pfdev); - - pm_runtime_set_active(pfdev->dev); - pm_runtime_get_sync(pfdev->dev); - pm_runtime_mark_last_busy(pfdev->dev); - pm_runtime_put_autosuspend(pfdev->dev); - err = panfrost_perfcnt_init(pfdev); if (err) goto err_out5; @@ -254,18 +246,22 @@ const char *panfrost_exception_name(struct panfrost_device *pfdev, u32 exception return "UNKNOWN"; } +void panfrost_device_reset(struct panfrost_device *pfdev) +{ + panfrost_gpu_soft_reset(pfdev); + + panfrost_gpu_power_on(pfdev); + panfrost_mmu_reset(pfdev); + panfrost_job_enable_interrupts(pfdev); +} + #ifdef CONFIG_PM int panfrost_device_resume(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct panfrost_device *pfdev = platform_get_drvdata(pdev); - panfrost_gpu_soft_reset(pfdev); - - /* TODO: Re-enable all other address spaces */ - panfrost_gpu_power_on(pfdev); - panfrost_mmu_enable(pfdev, 0); - panfrost_job_enable_interrupts(pfdev); + panfrost_device_reset(pfdev); panfrost_devfreq_resume(pfdev); return 0; diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h index 83cc01cafde1..9c39b9794811 100644 --- a/drivers/gpu/drm/panfrost/panfrost_device.h +++ b/drivers/gpu/drm/panfrost/panfrost_device.h @@ -5,6 +5,8 @@ #ifndef __PANFROST_DEVICE_H__ #define __PANFROST_DEVICE_H__ +#include <linux/atomic.h> +#include <linux/io-pgtable.h> #include <linux/spinlock.h> #include <drm/drm_device.h> #include <drm/drm_mm.h> @@ -43,6 +45,7 @@ struct panfrost_features { u32 js_features[16]; u32 nr_core_groups; + u32 thread_tls_alloc; unsigned long hw_features[64 / BITS_PER_LONG]; unsigned long hw_issues[64 / BITS_PER_LONG]; @@ -60,11 +63,6 @@ struct panfrost_device { struct drm_device *ddev; struct platform_device *pdev; - spinlock_t hwaccess_lock; - - struct drm_mm mm; - spinlock_t mm_lock; - void __iomem *iomem; struct clk *clock; struct clk *bus_clock; @@ -73,7 +71,11 @@ struct panfrost_device { struct panfrost_features features; - struct panfrost_mmu *mmu; + spinlock_t as_lock; + unsigned long as_in_use_mask; + unsigned long as_alloc_mask; + struct list_head as_lru_list; + struct panfrost_job_slot *js; struct panfrost_job *jobs[NUM_JOB_SLOTS]; @@ -84,6 +86,10 @@ struct panfrost_device { struct mutex sched_lock; struct mutex reset_lock; + struct mutex shrinker_lock; + struct list_head shrinker_list; + struct shrinker shrinker; + struct { struct devfreq *devfreq; struct thermal_cooling_device *cooling; @@ -93,10 +99,22 @@ struct panfrost_device { } devfreq; }; +struct panfrost_mmu { + struct io_pgtable_cfg pgtbl_cfg; + struct io_pgtable_ops *pgtbl_ops; + int as; + atomic_t as_count; + struct list_head list; +}; + struct panfrost_file_priv { struct panfrost_device *pfdev; struct drm_sched_entity sched_entity[NUM_JOB_SLOTS]; + + struct panfrost_mmu mmu; + struct drm_mm mm; + spinlock_t mm_lock; }; static inline struct panfrost_device *to_panfrost_device(struct drm_device *ddev) @@ -127,6 +145,7 @@ int panfrost_unstable_ioctl_check(void); int panfrost_device_init(struct panfrost_device *pfdev); void panfrost_device_fini(struct panfrost_device *pfdev); +void panfrost_device_reset(struct panfrost_device *pfdev); int panfrost_device_resume(struct device *dev); int panfrost_device_suspend(struct device *dev); diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c index 85b4b51b6a0d..bc2ddeb55f5d 100644 --- a/drivers/gpu/drm/panfrost/panfrost_drv.c +++ b/drivers/gpu/drm/panfrost/panfrost_drv.c @@ -32,10 +32,42 @@ static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct if (param->pad != 0) return -EINVAL; +#define PANFROST_FEATURE(name, member) \ + case DRM_PANFROST_PARAM_ ## name: \ + param->value = pfdev->features.member; \ + break +#define PANFROST_FEATURE_ARRAY(name, member, max) \ + case DRM_PANFROST_PARAM_ ## name ## 0 ... \ + DRM_PANFROST_PARAM_ ## name ## max: \ + param->value = pfdev->features.member[param->param - \ + DRM_PANFROST_PARAM_ ## name ## 0]; \ + break + switch (param->param) { - case DRM_PANFROST_PARAM_GPU_PROD_ID: - param->value = pfdev->features.id; - break; + PANFROST_FEATURE(GPU_PROD_ID, id); + PANFROST_FEATURE(GPU_REVISION, revision); + PANFROST_FEATURE(SHADER_PRESENT, shader_present); + PANFROST_FEATURE(TILER_PRESENT, tiler_present); + PANFROST_FEATURE(L2_PRESENT, l2_present); + PANFROST_FEATURE(STACK_PRESENT, stack_present); + PANFROST_FEATURE(AS_PRESENT, as_present); + PANFROST_FEATURE(JS_PRESENT, js_present); + PANFROST_FEATURE(L2_FEATURES, l2_features); + PANFROST_FEATURE(CORE_FEATURES, core_features); + PANFROST_FEATURE(TILER_FEATURES, tiler_features); + PANFROST_FEATURE(MEM_FEATURES, mem_features); + PANFROST_FEATURE(MMU_FEATURES, mmu_features); + PANFROST_FEATURE(THREAD_FEATURES, thread_features); + PANFROST_FEATURE(MAX_THREADS, max_threads); + PANFROST_FEATURE(THREAD_MAX_WORKGROUP_SZ, + thread_max_workgroup_sz); + PANFROST_FEATURE(THREAD_MAX_BARRIER_SZ, + thread_max_barrier_sz); + PANFROST_FEATURE(COHERENCY_FEATURES, coherency_features); + PANFROST_FEATURE_ARRAY(TEXTURE_FEATURES, texture_features, 3); + PANFROST_FEATURE_ARRAY(JS_FEATURES, js_features, 15); + PANFROST_FEATURE(NR_CORE_GROUPS, nr_core_groups); + PANFROST_FEATURE(THREAD_TLS_ALLOC, thread_tls_alloc); default: return -EINVAL; } @@ -46,29 +78,26 @@ static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data, struct drm_file *file) { - int ret; - struct drm_gem_shmem_object *shmem; + struct panfrost_gem_object *bo; struct drm_panfrost_create_bo *args = data; - if (!args->size || args->flags || args->pad) + if (!args->size || args->pad || + (args->flags & ~(PANFROST_BO_NOEXEC | PANFROST_BO_HEAP))) return -EINVAL; - shmem = drm_gem_shmem_create_with_handle(file, dev, args->size, - &args->handle); - if (IS_ERR(shmem)) - return PTR_ERR(shmem); + /* Heaps should never be executable */ + if ((args->flags & PANFROST_BO_HEAP) && + !(args->flags & PANFROST_BO_NOEXEC)) + return -EINVAL; - ret = panfrost_mmu_map(to_panfrost_bo(&shmem->base)); - if (ret) - goto err_free; + bo = panfrost_gem_create_with_handle(file, dev, args->size, args->flags, + &args->handle); + if (IS_ERR(bo)) + return PTR_ERR(bo); - args->offset = to_panfrost_bo(&shmem->base)->node.start << PAGE_SHIFT; + args->offset = bo->node.start << PAGE_SHIFT; return 0; - -err_free: - drm_gem_handle_delete(file, args->handle); - return ret; } /** @@ -245,7 +274,7 @@ panfrost_ioctl_wait_bo(struct drm_device *dev, void *data, if (!gem_obj) return -ENOENT; - ret = reservation_object_wait_timeout_rcu(gem_obj->resv, true, + ret = dma_resv_wait_timeout_rcu(gem_obj->resv, true, true, timeout); if (!ret) ret = timeout ? -ETIMEDOUT : -EBUSY; @@ -273,6 +302,10 @@ static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data, return -ENOENT; } + /* Don't allow mmapping of heap objects as pages are not pinned. */ + if (to_panfrost_bo(gem_obj)->is_heap) + return -EINVAL; + ret = drm_gem_create_mmap_offset(gem_obj); if (ret == 0) args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node); @@ -301,6 +334,38 @@ static int panfrost_ioctl_get_bo_offset(struct drm_device *dev, void *data, return 0; } +static int panfrost_ioctl_madvise(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct drm_panfrost_madvise *args = data; + struct panfrost_device *pfdev = dev->dev_private; + struct drm_gem_object *gem_obj; + + gem_obj = drm_gem_object_lookup(file_priv, args->handle); + if (!gem_obj) { + DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle); + return -ENOENT; + } + + args->retained = drm_gem_shmem_madvise(gem_obj, args->madv); + + if (args->retained) { + struct panfrost_gem_object *bo = to_panfrost_bo(gem_obj); + + mutex_lock(&pfdev->shrinker_lock); + + if (args->madv == PANFROST_MADV_DONTNEED) + list_add_tail(&bo->base.madv_list, &pfdev->shrinker_list); + else if (args->madv == PANFROST_MADV_WILLNEED) + list_del_init(&bo->base.madv_list); + + mutex_unlock(&pfdev->shrinker_lock); + } + + drm_gem_object_put_unlocked(gem_obj); + return 0; +} + int panfrost_unstable_ioctl_check(void) { if (!unstable_ioctls) @@ -309,9 +374,36 @@ int panfrost_unstable_ioctl_check(void) return 0; } +#define PFN_4G (SZ_4G >> PAGE_SHIFT) +#define PFN_4G_MASK (PFN_4G - 1) +#define PFN_16M (SZ_16M >> PAGE_SHIFT) + +static void panfrost_drm_mm_color_adjust(const struct drm_mm_node *node, + unsigned long color, + u64 *start, u64 *end) +{ + /* Executable buffers can't start or end on a 4GB boundary */ + if (!(color & PANFROST_BO_NOEXEC)) { + u64 next_seg; + + if ((*start & PFN_4G_MASK) == 0) + (*start)++; + + if ((*end & PFN_4G_MASK) == 0) + (*end)--; + + next_seg = ALIGN(*start, PFN_4G); + if (next_seg - *start <= PFN_16M) + *start = next_seg + 1; + + *end = min(*end, ALIGN(*start, PFN_4G) - 1); + } +} + static int panfrost_open(struct drm_device *dev, struct drm_file *file) { + int ret; struct panfrost_device *pfdev = dev->dev_private; struct panfrost_file_priv *panfrost_priv; @@ -322,7 +414,28 @@ panfrost_open(struct drm_device *dev, struct drm_file *file) panfrost_priv->pfdev = pfdev; file->driver_priv = panfrost_priv; - return panfrost_job_open(panfrost_priv); + spin_lock_init(&panfrost_priv->mm_lock); + + /* 4G enough for now. can be 48-bit */ + drm_mm_init(&panfrost_priv->mm, SZ_32M >> PAGE_SHIFT, (SZ_4G - SZ_32M) >> PAGE_SHIFT); + panfrost_priv->mm.color_adjust = panfrost_drm_mm_color_adjust; + + ret = panfrost_mmu_pgtable_alloc(panfrost_priv); + if (ret) + goto err_pgtable; + + ret = panfrost_job_open(panfrost_priv); + if (ret) + goto err_job; + + return 0; + +err_job: + panfrost_mmu_pgtable_free(panfrost_priv); +err_pgtable: + drm_mm_takedown(&panfrost_priv->mm); + kfree(panfrost_priv); + return ret; } static void @@ -333,6 +446,8 @@ panfrost_postclose(struct drm_device *dev, struct drm_file *file) panfrost_perfcnt_close(panfrost_priv); panfrost_job_close(panfrost_priv); + panfrost_mmu_pgtable_free(panfrost_priv); + drm_mm_takedown(&panfrost_priv->mm); kfree(panfrost_priv); } @@ -352,13 +467,18 @@ static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = { PANFROST_IOCTL(GET_BO_OFFSET, get_bo_offset, DRM_RENDER_ALLOW), PANFROST_IOCTL(PERFCNT_ENABLE, perfcnt_enable, DRM_RENDER_ALLOW), PANFROST_IOCTL(PERFCNT_DUMP, perfcnt_dump, DRM_RENDER_ALLOW), + PANFROST_IOCTL(MADVISE, madvise, DRM_RENDER_ALLOW), }; DEFINE_DRM_GEM_SHMEM_FOPS(panfrost_drm_driver_fops); +/* + * Panfrost driver version: + * - 1.0 - initial interface + * - 1.1 - adds HEAP and NOEXEC flags for CREATE_BO + */ static struct drm_driver panfrost_drm_driver = { - .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_PRIME | - DRIVER_SYNCOBJ, + .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ, .open = panfrost_open, .postclose = panfrost_postclose, .ioctls = panfrost_drm_driver_ioctls, @@ -368,7 +488,7 @@ static struct drm_driver panfrost_drm_driver = { .desc = "panfrost DRM", .date = "20180908", .major = 1, - .minor = 0, + .minor = 1, .gem_create_object = panfrost_gem_create_object, .prime_handle_to_fd = drm_gem_prime_handle_to_fd, @@ -400,14 +520,8 @@ static int panfrost_probe(struct platform_device *pdev) ddev->dev_private = pfdev; pfdev->ddev = ddev; - spin_lock_init(&pfdev->mm_lock); - - /* 4G enough for now. can be 48-bit */ - drm_mm_init(&pfdev->mm, SZ_32M >> PAGE_SHIFT, (SZ_4G - SZ_32M) >> PAGE_SHIFT); - - pm_runtime_use_autosuspend(pfdev->dev); - pm_runtime_set_autosuspend_delay(pfdev->dev, 50); /* ~3 frames */ - pm_runtime_enable(pfdev->dev); + mutex_init(&pfdev->shrinker_lock); + INIT_LIST_HEAD(&pfdev->shrinker_list); err = panfrost_device_init(pfdev); if (err) { @@ -423,16 +537,26 @@ static int panfrost_probe(struct platform_device *pdev) goto err_out1; } + pm_runtime_set_active(pfdev->dev); + pm_runtime_mark_last_busy(pfdev->dev); + pm_runtime_enable(pfdev->dev); + pm_runtime_set_autosuspend_delay(pfdev->dev, 50); /* ~3 frames */ + pm_runtime_use_autosuspend(pfdev->dev); + /* * Register the DRM device with the core and the connectors with * sysfs */ err = drm_dev_register(ddev, 0); if (err < 0) - goto err_out1; + goto err_out2; + + panfrost_gem_shrinker_init(ddev); return 0; +err_out2: + panfrost_devfreq_fini(pfdev); err_out1: panfrost_device_fini(pfdev); err_out0: @@ -447,10 +571,14 @@ static int panfrost_remove(struct platform_device *pdev) struct drm_device *ddev = pfdev->ddev; drm_dev_unregister(ddev); + panfrost_gem_shrinker_cleanup(ddev); + pm_runtime_get_sync(pfdev->dev); - pm_runtime_put_sync_autosuspend(pfdev->dev); - pm_runtime_disable(pfdev->dev); + panfrost_devfreq_fini(pfdev); panfrost_device_fini(pfdev); + pm_runtime_put_sync_suspend(pfdev->dev); + pm_runtime_disable(pfdev->dev); + drm_dev_put(ddev); return 0; } diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c index b46416be5a54..acb07fe06580 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gem.c +++ b/drivers/gpu/drm/panfrost/panfrost_gem.c @@ -19,20 +19,95 @@ static void panfrost_gem_free_object(struct drm_gem_object *obj) struct panfrost_gem_object *bo = to_panfrost_bo(obj); struct panfrost_device *pfdev = obj->dev->dev_private; + if (bo->sgts) { + int i; + int n_sgt = bo->base.base.size / SZ_2M; + + for (i = 0; i < n_sgt; i++) { + if (bo->sgts[i].sgl) { + dma_unmap_sg(pfdev->dev, bo->sgts[i].sgl, + bo->sgts[i].nents, DMA_BIDIRECTIONAL); + sg_free_table(&bo->sgts[i]); + } + } + kfree(bo->sgts); + } + + mutex_lock(&pfdev->shrinker_lock); + if (!list_empty(&bo->base.madv_list)) + list_del(&bo->base.madv_list); + mutex_unlock(&pfdev->shrinker_lock); + + drm_gem_shmem_free_object(obj); +} + +static int panfrost_gem_open(struct drm_gem_object *obj, struct drm_file *file_priv) +{ + int ret; + size_t size = obj->size; + u64 align; + struct panfrost_gem_object *bo = to_panfrost_bo(obj); + unsigned long color = bo->noexec ? PANFROST_BO_NOEXEC : 0; + struct panfrost_file_priv *priv = file_priv->driver_priv; + + /* + * Executable buffers cannot cross a 16MB boundary as the program + * counter is 24-bits. We assume executable buffers will be less than + * 16MB and aligning executable buffers to their size will avoid + * crossing a 16MB boundary. + */ + if (!bo->noexec) + align = size >> PAGE_SHIFT; + else + align = size >= SZ_2M ? SZ_2M >> PAGE_SHIFT : 0; + + bo->mmu = &priv->mmu; + spin_lock(&priv->mm_lock); + ret = drm_mm_insert_node_generic(&priv->mm, &bo->node, + size >> PAGE_SHIFT, align, color, 0); + spin_unlock(&priv->mm_lock); + if (ret) + return ret; + + if (!bo->is_heap) { + ret = panfrost_mmu_map(bo); + if (ret) { + spin_lock(&priv->mm_lock); + drm_mm_remove_node(&bo->node); + spin_unlock(&priv->mm_lock); + } + } + return ret; +} + +static void panfrost_gem_close(struct drm_gem_object *obj, struct drm_file *file_priv) +{ + struct panfrost_gem_object *bo = to_panfrost_bo(obj); + struct panfrost_file_priv *priv = file_priv->driver_priv; + if (bo->is_mapped) panfrost_mmu_unmap(bo); - spin_lock(&pfdev->mm_lock); - drm_mm_remove_node(&bo->node); - spin_unlock(&pfdev->mm_lock); + spin_lock(&priv->mm_lock); + if (drm_mm_node_allocated(&bo->node)) + drm_mm_remove_node(&bo->node); + spin_unlock(&priv->mm_lock); +} - drm_gem_shmem_free_object(obj); +static int panfrost_gem_pin(struct drm_gem_object *obj) +{ + if (to_panfrost_bo(obj)->is_heap) + return -EINVAL; + + return drm_gem_shmem_pin(obj); } static const struct drm_gem_object_funcs panfrost_gem_funcs = { .free = panfrost_gem_free_object, + .open = panfrost_gem_open, + .close = panfrost_gem_close, .print_info = drm_gem_shmem_print_info, - .pin = drm_gem_shmem_pin, + .pin = panfrost_gem_pin, .unpin = drm_gem_shmem_unpin, .get_sg_table = drm_gem_shmem_get_sg_table, .vmap = drm_gem_shmem_vmap, @@ -50,10 +125,7 @@ static const struct drm_gem_object_funcs panfrost_gem_funcs = { */ struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t size) { - int ret; - struct panfrost_device *pfdev = dev->dev_private; struct panfrost_gem_object *obj; - u64 align; obj = kzalloc(sizeof(*obj), GFP_KERNEL); if (!obj) @@ -61,21 +133,42 @@ struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t obj->base.base.funcs = &panfrost_gem_funcs; - size = roundup(size, PAGE_SIZE); - align = size >= SZ_2M ? SZ_2M >> PAGE_SHIFT : 0; + return &obj->base.base; +} - spin_lock(&pfdev->mm_lock); - ret = drm_mm_insert_node_generic(&pfdev->mm, &obj->node, - size >> PAGE_SHIFT, align, 0, 0); - spin_unlock(&pfdev->mm_lock); +struct panfrost_gem_object * +panfrost_gem_create_with_handle(struct drm_file *file_priv, + struct drm_device *dev, size_t size, + u32 flags, + uint32_t *handle) +{ + int ret; + struct drm_gem_shmem_object *shmem; + struct panfrost_gem_object *bo; + + /* Round up heap allocations to 2MB to keep fault handling simple */ + if (flags & PANFROST_BO_HEAP) + size = roundup(size, SZ_2M); + + shmem = drm_gem_shmem_create(dev, size); + if (IS_ERR(shmem)) + return ERR_CAST(shmem); + + bo = to_panfrost_bo(&shmem->base); + bo->noexec = !!(flags & PANFROST_BO_NOEXEC); + bo->is_heap = !!(flags & PANFROST_BO_HEAP); + + /* + * Allocate an id of idr table where the obj is registered + * and handle has the id what user can see. + */ + ret = drm_gem_handle_create(file_priv, &shmem->base, handle); + /* drop reference from allocate - handle holds it now. */ + drm_gem_object_put_unlocked(&shmem->base); if (ret) - goto free_obj; + return ERR_PTR(ret); - return &obj->base.base; - -free_obj: - kfree(obj); - return ERR_PTR(ret); + return bo; } struct drm_gem_object * @@ -84,17 +177,14 @@ panfrost_gem_prime_import_sg_table(struct drm_device *dev, struct sg_table *sgt) { struct drm_gem_object *obj; - struct panfrost_gem_object *pobj; + struct panfrost_gem_object *bo; obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt); if (IS_ERR(obj)) return ERR_CAST(obj); - pobj = to_panfrost_bo(obj); - - obj->resv = attach->dmabuf->resv; - - panfrost_mmu_map(pobj); + bo = to_panfrost_bo(obj); + bo->noexec = true; return obj; } diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.h b/drivers/gpu/drm/panfrost/panfrost_gem.h index 6dbcaba020fc..50920819cc16 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gem.h +++ b/drivers/gpu/drm/panfrost/panfrost_gem.h @@ -7,11 +7,17 @@ #include <drm/drm_gem_shmem_helper.h> #include <drm/drm_mm.h> +struct panfrost_mmu; + struct panfrost_gem_object { struct drm_gem_shmem_object base; + struct sg_table *sgts; + struct panfrost_mmu *mmu; struct drm_mm_node node; - bool is_mapped; + bool is_mapped :1; + bool noexec :1; + bool is_heap :1; }; static inline @@ -20,6 +26,12 @@ struct panfrost_gem_object *to_panfrost_bo(struct drm_gem_object *obj) return container_of(to_drm_gem_shmem_obj(obj), struct panfrost_gem_object, base); } +static inline +struct panfrost_gem_object *drm_mm_node_to_panfrost_bo(struct drm_mm_node *node) +{ + return container_of(node, struct panfrost_gem_object, node); +} + struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t size); struct drm_gem_object * @@ -27,4 +39,13 @@ panfrost_gem_prime_import_sg_table(struct drm_device *dev, struct dma_buf_attachment *attach, struct sg_table *sgt); +struct panfrost_gem_object * +panfrost_gem_create_with_handle(struct drm_file *file_priv, + struct drm_device *dev, size_t size, + u32 flags, + uint32_t *handle); + +void panfrost_gem_shrinker_init(struct drm_device *dev); +void panfrost_gem_shrinker_cleanup(struct drm_device *dev); + #endif /* __PANFROST_GEM_H__ */ diff --git a/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c b/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c new file mode 100644 index 000000000000..458f0fa68111 --- /dev/null +++ b/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2019 Arm Ltd. + * + * Based on msm_gem_freedreno.c: + * Copyright (C) 2016 Red Hat + * Author: Rob Clark <robdclark@gmail.com> + */ + +#include <linux/list.h> + +#include <drm/drm_device.h> +#include <drm/drm_gem_shmem_helper.h> + +#include "panfrost_device.h" +#include "panfrost_gem.h" +#include "panfrost_mmu.h" + +static unsigned long +panfrost_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) +{ + struct panfrost_device *pfdev = + container_of(shrinker, struct panfrost_device, shrinker); + struct drm_gem_shmem_object *shmem; + unsigned long count = 0; + + if (!mutex_trylock(&pfdev->shrinker_lock)) + return 0; + + list_for_each_entry(shmem, &pfdev->shrinker_list, madv_list) { + if (drm_gem_shmem_is_purgeable(shmem)) + count += shmem->base.size >> PAGE_SHIFT; + } + + mutex_unlock(&pfdev->shrinker_lock); + + return count; +} + +static bool panfrost_gem_purge(struct drm_gem_object *obj) +{ + struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); + + if (!mutex_trylock(&shmem->pages_lock)) + return false; + + panfrost_mmu_unmap(to_panfrost_bo(obj)); + drm_gem_shmem_purge_locked(obj); + + mutex_unlock(&shmem->pages_lock); + return true; +} + +static unsigned long +panfrost_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) +{ + struct panfrost_device *pfdev = + container_of(shrinker, struct panfrost_device, shrinker); + struct drm_gem_shmem_object *shmem, *tmp; + unsigned long freed = 0; + + if (!mutex_trylock(&pfdev->shrinker_lock)) + return SHRINK_STOP; + + list_for_each_entry_safe(shmem, tmp, &pfdev->shrinker_list, madv_list) { + if (freed >= sc->nr_to_scan) + break; + if (drm_gem_shmem_is_purgeable(shmem) && + panfrost_gem_purge(&shmem->base)) { + freed += shmem->base.size >> PAGE_SHIFT; + list_del_init(&shmem->madv_list); + } + } + + mutex_unlock(&pfdev->shrinker_lock); + + if (freed > 0) + pr_info_ratelimited("Purging %lu bytes\n", freed << PAGE_SHIFT); + + return freed; +} + +/** + * panfrost_gem_shrinker_init - Initialize panfrost shrinker + * @dev: DRM device + * + * This function registers and sets up the panfrost shrinker. + */ +void panfrost_gem_shrinker_init(struct drm_device *dev) +{ + struct panfrost_device *pfdev = dev->dev_private; + pfdev->shrinker.count_objects = panfrost_gem_shrinker_count; + pfdev->shrinker.scan_objects = panfrost_gem_shrinker_scan; + pfdev->shrinker.seeks = DEFAULT_SEEKS; + WARN_ON(register_shrinker(&pfdev->shrinker)); +} + +/** + * panfrost_gem_shrinker_cleanup - Clean up panfrost shrinker + * @dev: DRM device + * + * This function unregisters the panfrost shrinker. + */ +void panfrost_gem_shrinker_cleanup(struct drm_device *dev) +{ + struct panfrost_device *pfdev = dev->dev_private; + + if (pfdev->shrinker.nr_deferred) { + unregister_shrinker(&pfdev->shrinker); + } +} diff --git a/drivers/gpu/drm/panfrost/panfrost_gpu.c b/drivers/gpu/drm/panfrost/panfrost_gpu.c index 20ab333fc925..f67ed925c0ef 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gpu.c +++ b/drivers/gpu/drm/panfrost/panfrost_gpu.c @@ -232,6 +232,8 @@ static void panfrost_gpu_init_features(struct panfrost_device *pfdev) pfdev->features.stack_present = gpu_read(pfdev, GPU_STACK_PRESENT_LO); pfdev->features.stack_present |= (u64)gpu_read(pfdev, GPU_STACK_PRESENT_HI) << 32; + pfdev->features.thread_tls_alloc = gpu_read(pfdev, GPU_THREAD_TLS_ALLOC); + gpu_id = gpu_read(pfdev, GPU_ID); pfdev->features.revision = gpu_id & 0xffff; pfdev->features.id = gpu_id >> 16; diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c index 9bb9260d9181..a58551668d9a 100644 --- a/drivers/gpu/drm/panfrost/panfrost_job.c +++ b/drivers/gpu/drm/panfrost/panfrost_job.c @@ -6,7 +6,7 @@ #include <linux/io.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> -#include <linux/reservation.h> +#include <linux/dma-resv.h> #include <drm/gpu_scheduler.h> #include <drm/panfrost_drm.h> @@ -141,7 +141,6 @@ static void panfrost_job_write_affinity(struct panfrost_device *pfdev, static void panfrost_job_hw_submit(struct panfrost_job *job, int js) { struct panfrost_device *pfdev = job->pfdev; - unsigned long flags; u32 cfg; u64 jc_head = job->jc; int ret; @@ -150,11 +149,14 @@ static void panfrost_job_hw_submit(struct panfrost_job *job, int js) if (ret < 0) return; - if (WARN_ON(job_read(pfdev, JS_COMMAND_NEXT(js)))) - goto end; + if (WARN_ON(job_read(pfdev, JS_COMMAND_NEXT(js)))) { + pm_runtime_put_sync_autosuspend(pfdev->dev); + return; + } + + cfg = panfrost_mmu_as_get(pfdev, &job->file_priv->mmu); panfrost_devfreq_record_transition(pfdev, js); - spin_lock_irqsave(&pfdev->hwaccess_lock, flags); job_write(pfdev, JS_HEAD_NEXT_LO(js), jc_head & 0xFFFFFFFF); job_write(pfdev, JS_HEAD_NEXT_HI(js), jc_head >> 32); @@ -163,8 +165,7 @@ static void panfrost_job_hw_submit(struct panfrost_job *job, int js) /* start MMU, medium priority, cache clean/flush on end, clean/flush on * start */ - /* TODO: different address spaces */ - cfg = JS_CONFIG_THREAD_PRI(8) | + cfg |= JS_CONFIG_THREAD_PRI(8) | JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE | JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE; @@ -184,12 +185,6 @@ static void panfrost_job_hw_submit(struct panfrost_job *job, int js) job, js, jc_head); job_write(pfdev, JS_COMMAND_NEXT(js), JS_COMMAND_START); - - spin_unlock_irqrestore(&pfdev->hwaccess_lock, flags); - -end: - pm_runtime_mark_last_busy(pfdev->dev); - pm_runtime_put_autosuspend(pfdev->dev); } static void panfrost_acquire_object_fences(struct drm_gem_object **bos, @@ -199,7 +194,7 @@ static void panfrost_acquire_object_fences(struct drm_gem_object **bos, int i; for (i = 0; i < bo_count; i++) - implicit_fences[i] = reservation_object_get_excl_rcu(bos[i]->resv); + implicit_fences[i] = dma_resv_get_excl_rcu(bos[i]->resv); } static void panfrost_attach_object_fences(struct drm_gem_object **bos, @@ -209,7 +204,7 @@ static void panfrost_attach_object_fences(struct drm_gem_object **bos, int i; for (i = 0; i < bo_count; i++) - reservation_object_add_excl_fence(bos[i]->resv, fence); + dma_resv_add_excl_fence(bos[i]->resv, fence); } int panfrost_job_push(struct panfrost_job *job) @@ -368,6 +363,7 @@ static void panfrost_job_timedout(struct drm_sched_job *sched_job) struct panfrost_job *job = to_panfrost_job(sched_job); struct panfrost_device *pfdev = job->pfdev; int js = panfrost_job_get_slot(job); + unsigned long flags; int i; /* @@ -377,8 +373,9 @@ static void panfrost_job_timedout(struct drm_sched_job *sched_job) if (dma_fence_is_signaled(job->done_fence)) return; - dev_err(pfdev->dev, "gpu sched timeout, js=%d, status=0x%x, head=0x%x, tail=0x%x, sched_job=%p", + dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sched_job=%p", js, + job_read(pfdev, JS_CONFIG(js)), job_read(pfdev, JS_STATUS(js)), job_read(pfdev, JS_HEAD_LO(js)), job_read(pfdev, JS_TAIL_LO(js)), @@ -392,15 +389,19 @@ static void panfrost_job_timedout(struct drm_sched_job *sched_job) if (sched_job) drm_sched_increase_karma(sched_job); + spin_lock_irqsave(&pfdev->js->job_lock, flags); + for (i = 0; i < NUM_JOB_SLOTS; i++) { + if (pfdev->jobs[i]) { + pm_runtime_put_noidle(pfdev->dev); + pfdev->jobs[i] = NULL; + } + } + spin_unlock_irqrestore(&pfdev->js->job_lock, flags); + /* panfrost_core_dump(pfdev); */ panfrost_devfreq_record_transition(pfdev, js); - panfrost_gpu_soft_reset(pfdev); - - /* TODO: Re-enable all other address spaces */ - panfrost_mmu_enable(pfdev, 0); - panfrost_gpu_power_on(pfdev); - panfrost_job_enable_interrupts(pfdev); + panfrost_device_reset(pfdev); for (i = 0; i < NUM_JOB_SLOTS; i++) drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched); @@ -453,8 +454,21 @@ static irqreturn_t panfrost_job_irq_handler(int irq, void *data) } if (status & JOB_INT_MASK_DONE(j)) { - panfrost_devfreq_record_transition(pfdev, j); - dma_fence_signal(pfdev->jobs[j]->done_fence); + struct panfrost_job *job; + + spin_lock(&pfdev->js->job_lock); + job = pfdev->jobs[j]; + /* Only NULL if job timeout occurred */ + if (job) { + pfdev->jobs[j] = NULL; + + panfrost_mmu_as_put(pfdev, &job->file_priv->mmu); + panfrost_devfreq_record_transition(pfdev, j); + + dma_fence_signal_locked(job->done_fence); + pm_runtime_put_autosuspend(pfdev->dev); + } + spin_unlock(&pfdev->js->job_lock); } status &= ~mask; diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index 6e8145c36e93..6010f9ee7c1f 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */ +#include <linux/atomic.h> #include <linux/bitfield.h> #include <linux/delay.h> +#include <linux/dma-mapping.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/iopoll.h> @@ -9,6 +11,7 @@ #include <linux/iommu.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> +#include <linux/shmem_fs.h> #include <linux/sizes.h> #include "panfrost_device.h" @@ -20,12 +23,6 @@ #define mmu_write(dev, reg, data) writel(data, dev->iomem + reg) #define mmu_read(dev, reg) readl(dev->iomem + reg) -struct panfrost_mmu { - struct io_pgtable_cfg pgtbl_cfg; - struct io_pgtable_ops *pgtbl_ops; - struct mutex lock; -}; - static int wait_ready(struct panfrost_device *pfdev, u32 as_nr) { int ret; @@ -83,13 +80,11 @@ static void lock_region(struct panfrost_device *pfdev, u32 as_nr, } -static int mmu_hw_do_operation(struct panfrost_device *pfdev, u32 as_nr, - u64 iova, size_t size, u32 op) +static int mmu_hw_do_operation_locked(struct panfrost_device *pfdev, int as_nr, + u64 iova, size_t size, u32 op) { - unsigned long flags; - int ret; - - spin_lock_irqsave(&pfdev->hwaccess_lock, flags); + if (as_nr < 0) + return 0; if (op != AS_COMMAND_UNLOCK) lock_region(pfdev, as_nr, iova, size); @@ -98,21 +93,29 @@ static int mmu_hw_do_operation(struct panfrost_device *pfdev, u32 as_nr, write_cmd(pfdev, as_nr, op); /* Wait for the flush to complete */ - ret = wait_ready(pfdev, as_nr); + return wait_ready(pfdev, as_nr); +} - spin_unlock_irqrestore(&pfdev->hwaccess_lock, flags); +static int mmu_hw_do_operation(struct panfrost_device *pfdev, + struct panfrost_mmu *mmu, + u64 iova, size_t size, u32 op) +{ + int ret; + spin_lock(&pfdev->as_lock); + ret = mmu_hw_do_operation_locked(pfdev, mmu->as, iova, size, op); + spin_unlock(&pfdev->as_lock); return ret; } -void panfrost_mmu_enable(struct panfrost_device *pfdev, u32 as_nr) +static void panfrost_mmu_enable(struct panfrost_device *pfdev, struct panfrost_mmu *mmu) { - struct io_pgtable_cfg *cfg = &pfdev->mmu->pgtbl_cfg; + int as_nr = mmu->as; + struct io_pgtable_cfg *cfg = &mmu->pgtbl_cfg; u64 transtab = cfg->arm_mali_lpae_cfg.transtab; u64 memattr = cfg->arm_mali_lpae_cfg.memattr; - mmu_write(pfdev, MMU_INT_CLEAR, ~0); - mmu_write(pfdev, MMU_INT_MASK, ~0); + mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0UL, AS_COMMAND_FLUSH_MEM); mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), transtab & 0xffffffffUL); mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), transtab >> 32); @@ -126,8 +129,10 @@ void panfrost_mmu_enable(struct panfrost_device *pfdev, u32 as_nr) write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE); } -static void mmu_disable(struct panfrost_device *pfdev, u32 as_nr) +static void panfrost_mmu_disable(struct panfrost_device *pfdev, u32 as_nr) { + mmu_hw_do_operation_locked(pfdev, as_nr, 0, ~0UL, AS_COMMAND_FLUSH_MEM); + mmu_write(pfdev, AS_TRANSTAB_LO(as_nr), 0); mmu_write(pfdev, AS_TRANSTAB_HI(as_nr), 0); @@ -137,6 +142,80 @@ static void mmu_disable(struct panfrost_device *pfdev, u32 as_nr) write_cmd(pfdev, as_nr, AS_COMMAND_UPDATE); } +u32 panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu) +{ + int as; + + spin_lock(&pfdev->as_lock); + + as = mmu->as; + if (as >= 0) { + int en = atomic_inc_return(&mmu->as_count); + WARN_ON(en >= NUM_JOB_SLOTS); + + list_move(&mmu->list, &pfdev->as_lru_list); + goto out; + } + + /* Check for a free AS */ + as = ffz(pfdev->as_alloc_mask); + if (!(BIT(as) & pfdev->features.as_present)) { + struct panfrost_mmu *lru_mmu; + + list_for_each_entry_reverse(lru_mmu, &pfdev->as_lru_list, list) { + if (!atomic_read(&lru_mmu->as_count)) + break; + } + WARN_ON(&lru_mmu->list == &pfdev->as_lru_list); + + list_del_init(&lru_mmu->list); + as = lru_mmu->as; + + WARN_ON(as < 0); + lru_mmu->as = -1; + } + + /* Assign the free or reclaimed AS to the FD */ + mmu->as = as; + set_bit(as, &pfdev->as_alloc_mask); + atomic_set(&mmu->as_count, 1); + list_add(&mmu->list, &pfdev->as_lru_list); + + dev_dbg(pfdev->dev, "Assigned AS%d to mmu %p, alloc_mask=%lx", as, mmu, pfdev->as_alloc_mask); + + panfrost_mmu_enable(pfdev, mmu); + +out: + spin_unlock(&pfdev->as_lock); + return as; +} + +void panfrost_mmu_as_put(struct panfrost_device *pfdev, struct panfrost_mmu *mmu) +{ + atomic_dec(&mmu->as_count); + WARN_ON(atomic_read(&mmu->as_count) < 0); +} + +void panfrost_mmu_reset(struct panfrost_device *pfdev) +{ + struct panfrost_mmu *mmu, *mmu_tmp; + + spin_lock(&pfdev->as_lock); + + pfdev->as_alloc_mask = 0; + + list_for_each_entry_safe(mmu, mmu_tmp, &pfdev->as_lru_list, list) { + mmu->as = -1; + atomic_set(&mmu->as_count, 0); + list_del_init(&mmu->list); + } + + spin_unlock(&pfdev->as_lock); + + mmu_write(pfdev, MMU_INT_CLEAR, ~0); + mmu_write(pfdev, MMU_INT_MASK, ~0); +} + static size_t get_pgsize(u64 addr, size_t size) { if (addr & (SZ_2M - 1) || size < SZ_2M) @@ -145,53 +224,69 @@ static size_t get_pgsize(u64 addr, size_t size) return SZ_2M; } -int panfrost_mmu_map(struct panfrost_gem_object *bo) +void panfrost_mmu_flush_range(struct panfrost_device *pfdev, + struct panfrost_mmu *mmu, + u64 iova, size_t size) { - struct drm_gem_object *obj = &bo->base.base; - struct panfrost_device *pfdev = to_panfrost_device(obj->dev); - struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops; - u64 iova = bo->node.start << PAGE_SHIFT; - unsigned int count; - struct scatterlist *sgl; - struct sg_table *sgt; - int ret; + if (mmu->as < 0) + return; - if (WARN_ON(bo->is_mapped)) - return 0; + pm_runtime_get_noresume(pfdev->dev); - sgt = drm_gem_shmem_get_pages_sgt(obj); - if (WARN_ON(IS_ERR(sgt))) - return PTR_ERR(sgt); + /* Flush the PTs only if we're already awake */ + if (pm_runtime_active(pfdev->dev)) + mmu_hw_do_operation(pfdev, mmu, iova, size, AS_COMMAND_FLUSH_PT); - ret = pm_runtime_get_sync(pfdev->dev); - if (ret < 0) - return ret; + pm_runtime_put_sync_autosuspend(pfdev->dev); +} - mutex_lock(&pfdev->mmu->lock); +static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu, + u64 iova, int prot, struct sg_table *sgt) +{ + unsigned int count; + struct scatterlist *sgl; + struct io_pgtable_ops *ops = mmu->pgtbl_ops; + u64 start_iova = iova; for_each_sg(sgt->sgl, sgl, sgt->nents, count) { unsigned long paddr = sg_dma_address(sgl); size_t len = sg_dma_len(sgl); - dev_dbg(pfdev->dev, "map: iova=%llx, paddr=%lx, len=%zx", iova, paddr, len); + dev_dbg(pfdev->dev, "map: as=%d, iova=%llx, paddr=%lx, len=%zx", mmu->as, iova, paddr, len); while (len) { size_t pgsize = get_pgsize(iova | paddr, len); - ops->map(ops, iova, paddr, pgsize, IOMMU_WRITE | IOMMU_READ); + ops->map(ops, iova, paddr, pgsize, prot); iova += pgsize; paddr += pgsize; len -= pgsize; } } - mmu_hw_do_operation(pfdev, 0, bo->node.start << PAGE_SHIFT, - bo->node.size << PAGE_SHIFT, AS_COMMAND_FLUSH_PT); + panfrost_mmu_flush_range(pfdev, mmu, start_iova, iova - start_iova); + + return 0; +} + +int panfrost_mmu_map(struct panfrost_gem_object *bo) +{ + struct drm_gem_object *obj = &bo->base.base; + struct panfrost_device *pfdev = to_panfrost_device(obj->dev); + struct sg_table *sgt; + int prot = IOMMU_READ | IOMMU_WRITE; - mutex_unlock(&pfdev->mmu->lock); + if (WARN_ON(bo->is_mapped)) + return 0; + + if (bo->noexec) + prot |= IOMMU_NOEXEC; - pm_runtime_mark_last_busy(pfdev->dev); - pm_runtime_put_autosuspend(pfdev->dev); + sgt = drm_gem_shmem_get_pages_sgt(obj); + if (WARN_ON(IS_ERR(sgt))) + return PTR_ERR(sgt); + + mmu_map_sg(pfdev, bo->mmu, bo->node.start << PAGE_SHIFT, prot, sgt); bo->is_mapped = true; return 0; @@ -201,51 +296,34 @@ void panfrost_mmu_unmap(struct panfrost_gem_object *bo) { struct drm_gem_object *obj = &bo->base.base; struct panfrost_device *pfdev = to_panfrost_device(obj->dev); - struct io_pgtable_ops *ops = pfdev->mmu->pgtbl_ops; + struct io_pgtable_ops *ops = bo->mmu->pgtbl_ops; u64 iova = bo->node.start << PAGE_SHIFT; size_t len = bo->node.size << PAGE_SHIFT; size_t unmapped_len = 0; - int ret; if (WARN_ON(!bo->is_mapped)) return; - dev_dbg(pfdev->dev, "unmap: iova=%llx, len=%zx", iova, len); - - ret = pm_runtime_get_sync(pfdev->dev); - if (ret < 0) - return; - - mutex_lock(&pfdev->mmu->lock); + dev_dbg(pfdev->dev, "unmap: as=%d, iova=%llx, len=%zx", bo->mmu->as, iova, len); while (unmapped_len < len) { size_t unmapped_page; size_t pgsize = get_pgsize(iova, len - unmapped_len); - unmapped_page = ops->unmap(ops, iova, pgsize, NULL); - if (!unmapped_page) - break; - - iova += unmapped_page; - unmapped_len += unmapped_page; + if (ops->iova_to_phys(ops, iova)) { + unmapped_page = ops->unmap(ops, iova, pgsize, NULL); + WARN_ON(unmapped_page != pgsize); + } + iova += pgsize; + unmapped_len += pgsize; } - mmu_hw_do_operation(pfdev, 0, bo->node.start << PAGE_SHIFT, - bo->node.size << PAGE_SHIFT, AS_COMMAND_FLUSH_PT); - - mutex_unlock(&pfdev->mmu->lock); - - pm_runtime_mark_last_busy(pfdev->dev); - pm_runtime_put_autosuspend(pfdev->dev); + panfrost_mmu_flush_range(pfdev, bo->mmu, bo->node.start << PAGE_SHIFT, len); bo->is_mapped = false; } static void mmu_tlb_inv_context_s1(void *cookie) -{ - struct panfrost_device *pfdev = cookie; - - mmu_hw_do_operation(pfdev, 0, 0, ~0UL, AS_COMMAND_FLUSH_MEM); -} +{} static void mmu_tlb_sync_context(void *cookie) { @@ -271,6 +349,167 @@ static const struct iommu_flush_ops mmu_tlb_ops = { .tlb_flush_leaf = mmu_tlb_flush_leaf, }; +int panfrost_mmu_pgtable_alloc(struct panfrost_file_priv *priv) +{ + struct panfrost_mmu *mmu = &priv->mmu; + struct panfrost_device *pfdev = priv->pfdev; + + INIT_LIST_HEAD(&mmu->list); + mmu->as = -1; + + mmu->pgtbl_cfg = (struct io_pgtable_cfg) { + .pgsize_bitmap = SZ_4K | SZ_2M, + .ias = FIELD_GET(0xff, pfdev->features.mmu_features), + .oas = FIELD_GET(0xff00, pfdev->features.mmu_features), + .tlb = &mmu_tlb_ops, + .iommu_dev = pfdev->dev, + }; + + mmu->pgtbl_ops = alloc_io_pgtable_ops(ARM_MALI_LPAE, &mmu->pgtbl_cfg, + priv); + if (!mmu->pgtbl_ops) + return -EINVAL; + + return 0; +} + +void panfrost_mmu_pgtable_free(struct panfrost_file_priv *priv) +{ + struct panfrost_device *pfdev = priv->pfdev; + struct panfrost_mmu *mmu = &priv->mmu; + + spin_lock(&pfdev->as_lock); + if (mmu->as >= 0) { + pm_runtime_get_noresume(pfdev->dev); + if (pm_runtime_active(pfdev->dev)) + panfrost_mmu_disable(pfdev, mmu->as); + pm_runtime_put_autosuspend(pfdev->dev); + + clear_bit(mmu->as, &pfdev->as_alloc_mask); + clear_bit(mmu->as, &pfdev->as_in_use_mask); + list_del(&mmu->list); + } + spin_unlock(&pfdev->as_lock); + + free_io_pgtable_ops(mmu->pgtbl_ops); +} + +static struct drm_mm_node *addr_to_drm_mm_node(struct panfrost_device *pfdev, int as, u64 addr) +{ + struct drm_mm_node *node = NULL; + u64 offset = addr >> PAGE_SHIFT; + struct panfrost_mmu *mmu; + + spin_lock(&pfdev->as_lock); + list_for_each_entry(mmu, &pfdev->as_lru_list, list) { + struct panfrost_file_priv *priv; + if (as != mmu->as) + continue; + + priv = container_of(mmu, struct panfrost_file_priv, mmu); + drm_mm_for_each_node(node, &priv->mm) { + if (offset >= node->start && offset < (node->start + node->size)) + goto out; + } + } + +out: + spin_unlock(&pfdev->as_lock); + return node; +} + +#define NUM_FAULT_PAGES (SZ_2M / PAGE_SIZE) + +int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, u64 addr) +{ + int ret, i; + struct drm_mm_node *node; + struct panfrost_gem_object *bo; + struct address_space *mapping; + pgoff_t page_offset; + struct sg_table *sgt; + struct page **pages; + + node = addr_to_drm_mm_node(pfdev, as, addr); + if (!node) + return -ENOENT; + + bo = drm_mm_node_to_panfrost_bo(node); + if (!bo->is_heap) { + dev_WARN(pfdev->dev, "matching BO is not heap type (GPU VA = %llx)", + node->start << PAGE_SHIFT); + return -EINVAL; + } + WARN_ON(bo->mmu->as != as); + + /* Assume 2MB alignment and size multiple */ + addr &= ~((u64)SZ_2M - 1); + page_offset = addr >> PAGE_SHIFT; + page_offset -= node->start; + + mutex_lock(&bo->base.pages_lock); + + if (!bo->base.pages) { + bo->sgts = kvmalloc_array(bo->base.base.size / SZ_2M, + sizeof(struct sg_table), GFP_KERNEL | __GFP_ZERO); + if (!bo->sgts) { + mutex_unlock(&bo->base.pages_lock); + return -ENOMEM; + } + + pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT, + sizeof(struct page *), GFP_KERNEL | __GFP_ZERO); + if (!pages) { + kfree(bo->sgts); + bo->sgts = NULL; + mutex_unlock(&bo->base.pages_lock); + return -ENOMEM; + } + bo->base.pages = pages; + bo->base.pages_use_count = 1; + } else + pages = bo->base.pages; + + mapping = bo->base.base.filp->f_mapping; + mapping_set_unevictable(mapping); + + for (i = page_offset; i < page_offset + NUM_FAULT_PAGES; i++) { + pages[i] = shmem_read_mapping_page(mapping, i); + if (IS_ERR(pages[i])) { + mutex_unlock(&bo->base.pages_lock); + ret = PTR_ERR(pages[i]); + goto err_pages; + } + } + + mutex_unlock(&bo->base.pages_lock); + + sgt = &bo->sgts[page_offset / (SZ_2M / PAGE_SIZE)]; + ret = sg_alloc_table_from_pages(sgt, pages + page_offset, + NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL); + if (ret) + goto err_pages; + + if (!dma_map_sg(pfdev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL)) { + ret = -EINVAL; + goto err_map; + } + + mmu_map_sg(pfdev, bo->mmu, addr, IOMMU_WRITE | IOMMU_READ | IOMMU_NOEXEC, sgt); + + bo->is_mapped = true; + + dev_dbg(pfdev->dev, "mapped page fault @ AS%d %llx", as, addr); + + return 0; + +err_map: + sg_free_table(sgt); +err_pages: + drm_gem_shmem_put_pages(&bo->base); + return ret; +} + static const char *access_type_name(struct panfrost_device *pfdev, u32 fault_status) { @@ -295,13 +534,19 @@ static const char *access_type_name(struct panfrost_device *pfdev, static irqreturn_t panfrost_mmu_irq_handler(int irq, void *data) { struct panfrost_device *pfdev = data; - u32 status = mmu_read(pfdev, MMU_INT_STAT); - int i; - if (!status) + if (!mmu_read(pfdev, MMU_INT_STAT)) return IRQ_NONE; - dev_err(pfdev->dev, "mmu irq status=%x\n", status); + mmu_write(pfdev, MMU_INT_MASK, 0); + return IRQ_WAKE_THREAD; +} + +static irqreturn_t panfrost_mmu_irq_handler_thread(int irq, void *data) +{ + struct panfrost_device *pfdev = data; + u32 status = mmu_read(pfdev, MMU_INT_RAWSTAT); + int i, ret; for (i = 0; status; i++) { u32 mask = BIT(i) | BIT(i + 16); @@ -323,6 +568,18 @@ static irqreturn_t panfrost_mmu_irq_handler(int irq, void *data) access_type = (fault_status >> 8) & 0x3; source_id = (fault_status >> 16); + /* Page fault only */ + if ((status & mask) == BIT(i)) { + WARN_ON(exception_type < 0xC1 || exception_type > 0xC4); + + ret = panfrost_mmu_map_fault_addr(pfdev, i, addr); + if (!ret) { + mmu_write(pfdev, MMU_INT_CLEAR, BIT(i)); + status &= ~mask; + continue; + } + } + /* terminal fault, print info about the fault */ dev_err(pfdev->dev, "Unhandled Page fault in AS%d at VA 0x%016llX\n" @@ -345,50 +602,26 @@ static irqreturn_t panfrost_mmu_irq_handler(int irq, void *data) status &= ~mask; } + mmu_write(pfdev, MMU_INT_MASK, ~0); return IRQ_HANDLED; }; int panfrost_mmu_init(struct panfrost_device *pfdev) { - struct io_pgtable_ops *pgtbl_ops; int err, irq; - pfdev->mmu = devm_kzalloc(pfdev->dev, sizeof(*pfdev->mmu), GFP_KERNEL); - if (!pfdev->mmu) - return -ENOMEM; - - mutex_init(&pfdev->mmu->lock); - irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "mmu"); if (irq <= 0) return -ENODEV; - err = devm_request_irq(pfdev->dev, irq, panfrost_mmu_irq_handler, - IRQF_SHARED, "mmu", pfdev); + err = devm_request_threaded_irq(pfdev->dev, irq, panfrost_mmu_irq_handler, + panfrost_mmu_irq_handler_thread, + IRQF_SHARED, "mmu", pfdev); if (err) { dev_err(pfdev->dev, "failed to request mmu irq"); return err; } - mmu_write(pfdev, MMU_INT_CLEAR, ~0); - mmu_write(pfdev, MMU_INT_MASK, ~0); - - pfdev->mmu->pgtbl_cfg = (struct io_pgtable_cfg) { - .pgsize_bitmap = SZ_4K | SZ_2M, - .ias = FIELD_GET(0xff, pfdev->features.mmu_features), - .oas = FIELD_GET(0xff00, pfdev->features.mmu_features), - .tlb = &mmu_tlb_ops, - .iommu_dev = pfdev->dev, - }; - - pgtbl_ops = alloc_io_pgtable_ops(ARM_MALI_LPAE, &pfdev->mmu->pgtbl_cfg, - pfdev); - if (!pgtbl_ops) - return -ENOMEM; - - pfdev->mmu->pgtbl_ops = pgtbl_ops; - - panfrost_mmu_enable(pfdev, 0); return 0; } @@ -396,7 +629,4 @@ int panfrost_mmu_init(struct panfrost_device *pfdev) void panfrost_mmu_fini(struct panfrost_device *pfdev) { mmu_write(pfdev, MMU_INT_MASK, 0); - mmu_disable(pfdev, 0); - - free_io_pgtable_ops(pfdev->mmu->pgtbl_ops); } diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.h b/drivers/gpu/drm/panfrost/panfrost_mmu.h index f5878d86a5ce..7c5b6775ae23 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.h +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.h @@ -5,13 +5,20 @@ #define __PANFROST_MMU_H__ struct panfrost_gem_object; +struct panfrost_file_priv; +struct panfrost_mmu; int panfrost_mmu_map(struct panfrost_gem_object *bo); void panfrost_mmu_unmap(struct panfrost_gem_object *bo); int panfrost_mmu_init(struct panfrost_device *pfdev); void panfrost_mmu_fini(struct panfrost_device *pfdev); +void panfrost_mmu_reset(struct panfrost_device *pfdev); -void panfrost_mmu_enable(struct panfrost_device *pfdev, u32 as_nr); +u32 panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu); +void panfrost_mmu_as_put(struct panfrost_device *pfdev, struct panfrost_mmu *mmu); + +int panfrost_mmu_pgtable_alloc(struct panfrost_file_priv *priv); +void panfrost_mmu_pgtable_free(struct panfrost_file_priv *priv); #endif |