diff options
Diffstat (limited to 'drivers/gpu/drm/i915/selftests')
43 files changed, 510 insertions, 10458 deletions
diff --git a/drivers/gpu/drm/i915/selftests/huge_gem_object.c b/drivers/gpu/drm/i915/selftests/huge_gem_object.c deleted file mode 100644 index 419fd4d6a8f0..000000000000 --- a/drivers/gpu/drm/i915/selftests/huge_gem_object.c +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright © 2016 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#include "huge_gem_object.h" - -static void huge_free_pages(struct drm_i915_gem_object *obj, - struct sg_table *pages) -{ - unsigned long nreal = obj->scratch / PAGE_SIZE; - struct scatterlist *sg; - - for (sg = pages->sgl; sg && nreal--; sg = __sg_next(sg)) - __free_page(sg_page(sg)); - - sg_free_table(pages); - kfree(pages); -} - -static int huge_get_pages(struct drm_i915_gem_object *obj) -{ -#define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) - const unsigned long nreal = obj->scratch / PAGE_SIZE; - const unsigned long npages = obj->base.size / PAGE_SIZE; - struct scatterlist *sg, *src, *end; - struct sg_table *pages; - unsigned long n; - - pages = kmalloc(sizeof(*pages), GFP); - if (!pages) - return -ENOMEM; - - if (sg_alloc_table(pages, npages, GFP)) { - kfree(pages); - return -ENOMEM; - } - - sg = pages->sgl; - for (n = 0; n < nreal; n++) { - struct page *page; - - page = alloc_page(GFP | __GFP_HIGHMEM); - if (!page) { - sg_mark_end(sg); - goto err; - } - - sg_set_page(sg, page, PAGE_SIZE, 0); - sg = __sg_next(sg); - } - if (nreal < npages) { - for (end = sg, src = pages->sgl; sg; sg = __sg_next(sg)) { - sg_set_page(sg, sg_page(src), PAGE_SIZE, 0); - src = __sg_next(src); - if (src == end) - src = pages->sgl; - } - } - - if (i915_gem_gtt_prepare_pages(obj, pages)) - goto err; - - __i915_gem_object_set_pages(obj, pages, PAGE_SIZE); - - return 0; - -err: - huge_free_pages(obj, pages); - - return -ENOMEM; -#undef GFP -} - -static void huge_put_pages(struct drm_i915_gem_object *obj, - struct sg_table *pages) -{ - i915_gem_gtt_finish_pages(obj, pages); - huge_free_pages(obj, pages); - - obj->mm.dirty = false; -} - -static const struct drm_i915_gem_object_ops huge_ops = { - .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | - I915_GEM_OBJECT_IS_SHRINKABLE, - .get_pages = huge_get_pages, - .put_pages = huge_put_pages, -}; - -struct drm_i915_gem_object * -huge_gem_object(struct drm_i915_private *i915, - phys_addr_t phys_size, - dma_addr_t dma_size) -{ - struct drm_i915_gem_object *obj; - unsigned int cache_level; - - GEM_BUG_ON(!phys_size || phys_size > dma_size); - GEM_BUG_ON(!IS_ALIGNED(phys_size, PAGE_SIZE)); - GEM_BUG_ON(!IS_ALIGNED(dma_size, I915_GTT_PAGE_SIZE)); - - if (overflows_type(dma_size, obj->base.size)) - return ERR_PTR(-E2BIG); - - obj = i915_gem_object_alloc(); - if (!obj) - return ERR_PTR(-ENOMEM); - - drm_gem_private_object_init(&i915->drm, &obj->base, dma_size); - i915_gem_object_init(obj, &huge_ops); - - obj->read_domains = I915_GEM_DOMAIN_CPU; - obj->write_domain = I915_GEM_DOMAIN_CPU; - cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE; - i915_gem_object_set_cache_coherency(obj, cache_level); - obj->scratch = phys_size; - - return obj; -} diff --git a/drivers/gpu/drm/i915/selftests/huge_gem_object.h b/drivers/gpu/drm/i915/selftests/huge_gem_object.h deleted file mode 100644 index a6133a9e8029..000000000000 --- a/drivers/gpu/drm/i915/selftests/huge_gem_object.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright © 2016 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#ifndef __HUGE_GEM_OBJECT_H -#define __HUGE_GEM_OBJECT_H - -struct drm_i915_gem_object * -huge_gem_object(struct drm_i915_private *i915, - phys_addr_t phys_size, - dma_addr_t dma_size); - -static inline phys_addr_t -huge_gem_object_phys_size(struct drm_i915_gem_object *obj) -{ - return obj->scratch; -} - -static inline dma_addr_t -huge_gem_object_dma_size(struct drm_i915_gem_object *obj) -{ - return obj->base.size; -} - -#endif /* !__HUGE_GEM_OBJECT_H */ diff --git a/drivers/gpu/drm/i915/selftests/huge_pages.c b/drivers/gpu/drm/i915/selftests/huge_pages.c deleted file mode 100644 index 90721b54e7ae..000000000000 --- a/drivers/gpu/drm/i915/selftests/huge_pages.c +++ /dev/null @@ -1,1792 +0,0 @@ -/* - * Copyright © 2017 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#include "../i915_selftest.h" - -#include <linux/prime_numbers.h> - -#include "mock_drm.h" -#include "i915_random.h" - -static const unsigned int page_sizes[] = { - I915_GTT_PAGE_SIZE_2M, - I915_GTT_PAGE_SIZE_64K, - I915_GTT_PAGE_SIZE_4K, -}; - -static unsigned int get_largest_page_size(struct drm_i915_private *i915, - u64 rem) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) { - unsigned int page_size = page_sizes[i]; - - if (HAS_PAGE_SIZES(i915, page_size) && rem >= page_size) - return page_size; - } - - return 0; -} - -static void huge_pages_free_pages(struct sg_table *st) -{ - struct scatterlist *sg; - - for (sg = st->sgl; sg; sg = __sg_next(sg)) { - if (sg_page(sg)) - __free_pages(sg_page(sg), get_order(sg->length)); - } - - sg_free_table(st); - kfree(st); -} - -static int get_huge_pages(struct drm_i915_gem_object *obj) -{ -#define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) - unsigned int page_mask = obj->mm.page_mask; - struct sg_table *st; - struct scatterlist *sg; - unsigned int sg_page_sizes; - u64 rem; - - st = kmalloc(sizeof(*st), GFP); - if (!st) - return -ENOMEM; - - if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) { - kfree(st); - return -ENOMEM; - } - - rem = obj->base.size; - sg = st->sgl; - st->nents = 0; - sg_page_sizes = 0; - - /* - * Our goal here is simple, we want to greedily fill the object from - * largest to smallest page-size, while ensuring that we use *every* - * page-size as per the given page-mask. - */ - do { - unsigned int bit = ilog2(page_mask); - unsigned int page_size = BIT(bit); - int order = get_order(page_size); - - do { - struct page *page; - - GEM_BUG_ON(order >= MAX_ORDER); - page = alloc_pages(GFP | __GFP_ZERO, order); - if (!page) - goto err; - - sg_set_page(sg, page, page_size, 0); - sg_page_sizes |= page_size; - st->nents++; - - rem -= page_size; - if (!rem) { - sg_mark_end(sg); - break; - } - - sg = __sg_next(sg); - } while ((rem - ((page_size-1) & page_mask)) >= page_size); - - page_mask &= (page_size-1); - } while (page_mask); - - if (i915_gem_gtt_prepare_pages(obj, st)) - goto err; - - obj->mm.madv = I915_MADV_DONTNEED; - - GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask); - __i915_gem_object_set_pages(obj, st, sg_page_sizes); - - return 0; - -err: - sg_set_page(sg, NULL, 0, 0); - sg_mark_end(sg); - huge_pages_free_pages(st); - - return -ENOMEM; -} - -static void put_huge_pages(struct drm_i915_gem_object *obj, - struct sg_table *pages) -{ - i915_gem_gtt_finish_pages(obj, pages); - huge_pages_free_pages(pages); - - obj->mm.dirty = false; - obj->mm.madv = I915_MADV_WILLNEED; -} - -static const struct drm_i915_gem_object_ops huge_page_ops = { - .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | - I915_GEM_OBJECT_IS_SHRINKABLE, - .get_pages = get_huge_pages, - .put_pages = put_huge_pages, -}; - -static struct drm_i915_gem_object * -huge_pages_object(struct drm_i915_private *i915, - u64 size, - unsigned int page_mask) -{ - struct drm_i915_gem_object *obj; - - GEM_BUG_ON(!size); - GEM_BUG_ON(!IS_ALIGNED(size, BIT(__ffs(page_mask)))); - - if (size >> PAGE_SHIFT > INT_MAX) - return ERR_PTR(-E2BIG); - - if (overflows_type(size, obj->base.size)) - return ERR_PTR(-E2BIG); - - obj = i915_gem_object_alloc(); - if (!obj) - return ERR_PTR(-ENOMEM); - - drm_gem_private_object_init(&i915->drm, &obj->base, size); - i915_gem_object_init(obj, &huge_page_ops); - - obj->write_domain = I915_GEM_DOMAIN_CPU; - obj->read_domains = I915_GEM_DOMAIN_CPU; - obj->cache_level = I915_CACHE_NONE; - - obj->mm.page_mask = page_mask; - - return obj; -} - -static int fake_get_huge_pages(struct drm_i915_gem_object *obj) -{ - struct drm_i915_private *i915 = to_i915(obj->base.dev); - const u64 max_len = rounddown_pow_of_two(UINT_MAX); - struct sg_table *st; - struct scatterlist *sg; - unsigned int sg_page_sizes; - u64 rem; - - st = kmalloc(sizeof(*st), GFP); - if (!st) - return -ENOMEM; - - if (sg_alloc_table(st, obj->base.size >> PAGE_SHIFT, GFP)) { - kfree(st); - return -ENOMEM; - } - - /* Use optimal page sized chunks to fill in the sg table */ - rem = obj->base.size; - sg = st->sgl; - st->nents = 0; - sg_page_sizes = 0; - do { - unsigned int page_size = get_largest_page_size(i915, rem); - unsigned int len = min(page_size * div_u64(rem, page_size), - max_len); - - GEM_BUG_ON(!page_size); - - sg->offset = 0; - sg->length = len; - sg_dma_len(sg) = len; - sg_dma_address(sg) = page_size; - - sg_page_sizes |= len; - - st->nents++; - - rem -= len; - if (!rem) { - sg_mark_end(sg); - break; - } - - sg = sg_next(sg); - } while (1); - - i915_sg_trim(st); - - obj->mm.madv = I915_MADV_DONTNEED; - - __i915_gem_object_set_pages(obj, st, sg_page_sizes); - - return 0; -} - -static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj) -{ - struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct sg_table *st; - struct scatterlist *sg; - unsigned int page_size; - - st = kmalloc(sizeof(*st), GFP); - if (!st) - return -ENOMEM; - - if (sg_alloc_table(st, 1, GFP)) { - kfree(st); - return -ENOMEM; - } - - sg = st->sgl; - st->nents = 1; - - page_size = get_largest_page_size(i915, obj->base.size); - GEM_BUG_ON(!page_size); - - sg->offset = 0; - sg->length = obj->base.size; - sg_dma_len(sg) = obj->base.size; - sg_dma_address(sg) = page_size; - - obj->mm.madv = I915_MADV_DONTNEED; - - __i915_gem_object_set_pages(obj, st, sg->length); - - return 0; -#undef GFP -} - -static void fake_free_huge_pages(struct drm_i915_gem_object *obj, - struct sg_table *pages) -{ - sg_free_table(pages); - kfree(pages); -} - -static void fake_put_huge_pages(struct drm_i915_gem_object *obj, - struct sg_table *pages) -{ - fake_free_huge_pages(obj, pages); - obj->mm.dirty = false; - obj->mm.madv = I915_MADV_WILLNEED; -} - -static const struct drm_i915_gem_object_ops fake_ops = { - .flags = I915_GEM_OBJECT_IS_SHRINKABLE, - .get_pages = fake_get_huge_pages, - .put_pages = fake_put_huge_pages, -}; - -static const struct drm_i915_gem_object_ops fake_ops_single = { - .flags = I915_GEM_OBJECT_IS_SHRINKABLE, - .get_pages = fake_get_huge_pages_single, - .put_pages = fake_put_huge_pages, -}; - -static struct drm_i915_gem_object * -fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single) -{ - struct drm_i915_gem_object *obj; - - GEM_BUG_ON(!size); - GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); - - if (size >> PAGE_SHIFT > UINT_MAX) - return ERR_PTR(-E2BIG); - - if (overflows_type(size, obj->base.size)) - return ERR_PTR(-E2BIG); - - obj = i915_gem_object_alloc(); - if (!obj) - return ERR_PTR(-ENOMEM); - - drm_gem_private_object_init(&i915->drm, &obj->base, size); - - if (single) - i915_gem_object_init(obj, &fake_ops_single); - else - i915_gem_object_init(obj, &fake_ops); - - obj->write_domain = I915_GEM_DOMAIN_CPU; - obj->read_domains = I915_GEM_DOMAIN_CPU; - obj->cache_level = I915_CACHE_NONE; - - return obj; -} - -static int igt_check_page_sizes(struct i915_vma *vma) -{ - struct drm_i915_private *i915 = vma->vm->i915; - unsigned int supported = INTEL_INFO(i915)->page_sizes; - struct drm_i915_gem_object *obj = vma->obj; - int err = 0; - - if (!HAS_PAGE_SIZES(i915, vma->page_sizes.sg)) { - pr_err("unsupported page_sizes.sg=%u, supported=%u\n", - vma->page_sizes.sg & ~supported, supported); - err = -EINVAL; - } - - if (!HAS_PAGE_SIZES(i915, vma->page_sizes.gtt)) { - pr_err("unsupported page_sizes.gtt=%u, supported=%u\n", - vma->page_sizes.gtt & ~supported, supported); - err = -EINVAL; - } - - if (vma->page_sizes.phys != obj->mm.page_sizes.phys) { - pr_err("vma->page_sizes.phys(%u) != obj->mm.page_sizes.phys(%u)\n", - vma->page_sizes.phys, obj->mm.page_sizes.phys); - err = -EINVAL; - } - - if (vma->page_sizes.sg != obj->mm.page_sizes.sg) { - pr_err("vma->page_sizes.sg(%u) != obj->mm.page_sizes.sg(%u)\n", - vma->page_sizes.sg, obj->mm.page_sizes.sg); - err = -EINVAL; - } - - if (obj->mm.page_sizes.gtt) { - pr_err("obj->page_sizes.gtt(%u) should never be set\n", - obj->mm.page_sizes.gtt); - err = -EINVAL; - } - - return err; -} - -static int igt_mock_exhaust_device_supported_pages(void *arg) -{ - struct i915_hw_ppgtt *ppgtt = arg; - struct drm_i915_private *i915 = ppgtt->vm.i915; - unsigned int saved_mask = INTEL_INFO(i915)->page_sizes; - struct drm_i915_gem_object *obj; - struct i915_vma *vma; - int i, j, single; - int err; - - /* - * Sanity check creating objects with every valid page support - * combination for our mock device. - */ - - for (i = 1; i < BIT(ARRAY_SIZE(page_sizes)); i++) { - unsigned int combination = 0; - - for (j = 0; j < ARRAY_SIZE(page_sizes); j++) { - if (i & BIT(j)) - combination |= page_sizes[j]; - } - - mkwrite_device_info(i915)->page_sizes = combination; - - for (single = 0; single <= 1; ++single) { - obj = fake_huge_pages_object(i915, combination, !!single); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto out_device; - } - - if (obj->base.size != combination) { - pr_err("obj->base.size=%zu, expected=%u\n", - obj->base.size, combination); - err = -EINVAL; - goto out_put; - } - - vma = i915_vma_instance(obj, &ppgtt->vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto out_put; - } - - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) - goto out_close; - - err = igt_check_page_sizes(vma); - - if (vma->page_sizes.sg != combination) { - pr_err("page_sizes.sg=%u, expected=%u\n", - vma->page_sizes.sg, combination); - err = -EINVAL; - } - - i915_vma_unpin(vma); - i915_vma_close(vma); - - i915_gem_object_put(obj); - - if (err) - goto out_device; - } - } - - goto out_device; - -out_close: - i915_vma_close(vma); -out_put: - i915_gem_object_put(obj); -out_device: - mkwrite_device_info(i915)->page_sizes = saved_mask; - - return err; -} - -static int igt_mock_ppgtt_misaligned_dma(void *arg) -{ - struct i915_hw_ppgtt *ppgtt = arg; - struct drm_i915_private *i915 = ppgtt->vm.i915; - unsigned long supported = INTEL_INFO(i915)->page_sizes; - struct drm_i915_gem_object *obj; - int bit; - int err; - - /* - * Sanity check dma misalignment for huge pages -- the dma addresses we - * insert into the paging structures need to always respect the page - * size alignment. - */ - - bit = ilog2(I915_GTT_PAGE_SIZE_64K); - - for_each_set_bit_from(bit, &supported, - ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) { - IGT_TIMEOUT(end_time); - unsigned int page_size = BIT(bit); - unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; - unsigned int offset; - unsigned int size = - round_up(page_size, I915_GTT_PAGE_SIZE_2M) << 1; - struct i915_vma *vma; - - obj = fake_huge_pages_object(i915, size, true); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - if (obj->base.size != size) { - pr_err("obj->base.size=%zu, expected=%u\n", - obj->base.size, size); - err = -EINVAL; - goto out_put; - } - - err = i915_gem_object_pin_pages(obj); - if (err) - goto out_put; - - /* Force the page size for this object */ - obj->mm.page_sizes.sg = page_size; - - vma = i915_vma_instance(obj, &ppgtt->vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto out_unpin; - } - - err = i915_vma_pin(vma, 0, 0, flags); - if (err) { - i915_vma_close(vma); - goto out_unpin; - } - - - err = igt_check_page_sizes(vma); - - if (vma->page_sizes.gtt != page_size) { - pr_err("page_sizes.gtt=%u, expected %u\n", - vma->page_sizes.gtt, page_size); - err = -EINVAL; - } - - i915_vma_unpin(vma); - - if (err) { - i915_vma_close(vma); - goto out_unpin; - } - - /* - * Try all the other valid offsets until the next - * boundary -- should always fall back to using 4K - * pages. - */ - for (offset = 4096; offset < page_size; offset += 4096) { - err = i915_vma_unbind(vma); - if (err) { - i915_vma_close(vma); - goto out_unpin; - } - - err = i915_vma_pin(vma, 0, 0, flags | offset); - if (err) { - i915_vma_close(vma); - goto out_unpin; - } - - err = igt_check_page_sizes(vma); - - if (vma->page_sizes.gtt != I915_GTT_PAGE_SIZE_4K) { - pr_err("page_sizes.gtt=%u, expected %llu\n", - vma->page_sizes.gtt, I915_GTT_PAGE_SIZE_4K); - err = -EINVAL; - } - - i915_vma_unpin(vma); - - if (err) { - i915_vma_close(vma); - goto out_unpin; - } - - if (igt_timeout(end_time, - "%s timed out at offset %x with page-size %x\n", - __func__, offset, page_size)) - break; - } - - i915_vma_close(vma); - - i915_gem_object_unpin_pages(obj); - __i915_gem_object_put_pages(obj, I915_MM_NORMAL); - i915_gem_object_put(obj); - } - - return 0; - -out_unpin: - i915_gem_object_unpin_pages(obj); -out_put: - i915_gem_object_put(obj); - - return err; -} - -static void close_object_list(struct list_head *objects, - struct i915_hw_ppgtt *ppgtt) -{ - struct drm_i915_gem_object *obj, *on; - - list_for_each_entry_safe(obj, on, objects, st_link) { - struct i915_vma *vma; - - vma = i915_vma_instance(obj, &ppgtt->vm, NULL); - if (!IS_ERR(vma)) - i915_vma_close(vma); - - list_del(&obj->st_link); - i915_gem_object_unpin_pages(obj); - __i915_gem_object_put_pages(obj, I915_MM_NORMAL); - i915_gem_object_put(obj); - } -} - -static int igt_mock_ppgtt_huge_fill(void *arg) -{ - struct i915_hw_ppgtt *ppgtt = arg; - struct drm_i915_private *i915 = ppgtt->vm.i915; - unsigned long max_pages = ppgtt->vm.total >> PAGE_SHIFT; - unsigned long page_num; - bool single = false; - LIST_HEAD(objects); - IGT_TIMEOUT(end_time); - int err = -ENODEV; - - for_each_prime_number_from(page_num, 1, max_pages) { - struct drm_i915_gem_object *obj; - u64 size = page_num << PAGE_SHIFT; - struct i915_vma *vma; - unsigned int expected_gtt = 0; - int i; - - obj = fake_huge_pages_object(i915, size, single); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - break; - } - - if (obj->base.size != size) { - pr_err("obj->base.size=%zd, expected=%llu\n", - obj->base.size, size); - i915_gem_object_put(obj); - err = -EINVAL; - break; - } - - err = i915_gem_object_pin_pages(obj); - if (err) { - i915_gem_object_put(obj); - break; - } - - list_add(&obj->st_link, &objects); - - vma = i915_vma_instance(obj, &ppgtt->vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - break; - } - - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) - break; - - err = igt_check_page_sizes(vma); - if (err) { - i915_vma_unpin(vma); - break; - } - - /* - * Figure out the expected gtt page size knowing that we go from - * largest to smallest page size sg chunks, and that we align to - * the largest page size. - */ - for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) { - unsigned int page_size = page_sizes[i]; - - if (HAS_PAGE_SIZES(i915, page_size) && - size >= page_size) { - expected_gtt |= page_size; - size &= page_size-1; - } - } - - GEM_BUG_ON(!expected_gtt); - GEM_BUG_ON(size); - - if (expected_gtt & I915_GTT_PAGE_SIZE_4K) - expected_gtt &= ~I915_GTT_PAGE_SIZE_64K; - - i915_vma_unpin(vma); - - if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) { - if (!IS_ALIGNED(vma->node.start, - I915_GTT_PAGE_SIZE_2M)) { - pr_err("node.start(%llx) not aligned to 2M\n", - vma->node.start); - err = -EINVAL; - break; - } - - if (!IS_ALIGNED(vma->node.size, - I915_GTT_PAGE_SIZE_2M)) { - pr_err("node.size(%llx) not aligned to 2M\n", - vma->node.size); - err = -EINVAL; - break; - } - } - - if (vma->page_sizes.gtt != expected_gtt) { - pr_err("gtt=%u, expected=%u, size=%zd, single=%s\n", - vma->page_sizes.gtt, expected_gtt, - obj->base.size, yesno(!!single)); - err = -EINVAL; - break; - } - - if (igt_timeout(end_time, - "%s timed out at size %zd\n", - __func__, obj->base.size)) - break; - - single = !single; - } - - close_object_list(&objects, ppgtt); - - if (err == -ENOMEM || err == -ENOSPC) - err = 0; - - return err; -} - -static int igt_mock_ppgtt_64K(void *arg) -{ - struct i915_hw_ppgtt *ppgtt = arg; - struct drm_i915_private *i915 = ppgtt->vm.i915; - struct drm_i915_gem_object *obj; - const struct object_info { - unsigned int size; - unsigned int gtt; - unsigned int offset; - } objects[] = { - /* Cases with forced padding/alignment */ - { - .size = SZ_64K, - .gtt = I915_GTT_PAGE_SIZE_64K, - .offset = 0, - }, - { - .size = SZ_64K + SZ_4K, - .gtt = I915_GTT_PAGE_SIZE_4K, - .offset = 0, - }, - { - .size = SZ_64K - SZ_4K, - .gtt = I915_GTT_PAGE_SIZE_4K, - .offset = 0, - }, - { - .size = SZ_2M, - .gtt = I915_GTT_PAGE_SIZE_64K, - .offset = 0, - }, - { - .size = SZ_2M - SZ_4K, - .gtt = I915_GTT_PAGE_SIZE_4K, - .offset = 0, - }, - { - .size = SZ_2M + SZ_4K, - .gtt = I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_4K, - .offset = 0, - }, - { - .size = SZ_2M + SZ_64K, - .gtt = I915_GTT_PAGE_SIZE_64K, - .offset = 0, - }, - { - .size = SZ_2M - SZ_64K, - .gtt = I915_GTT_PAGE_SIZE_64K, - .offset = 0, - }, - /* Try without any forced padding/alignment */ - { - .size = SZ_64K, - .offset = SZ_2M, - .gtt = I915_GTT_PAGE_SIZE_4K, - }, - { - .size = SZ_128K, - .offset = SZ_2M - SZ_64K, - .gtt = I915_GTT_PAGE_SIZE_4K, - }, - }; - struct i915_vma *vma; - int i, single; - int err; - - /* - * Sanity check some of the trickiness with 64K pages -- either we can - * safely mark the whole page-table(2M block) as 64K, or we have to - * always fallback to 4K. - */ - - if (!HAS_PAGE_SIZES(i915, I915_GTT_PAGE_SIZE_64K)) - return 0; - - for (i = 0; i < ARRAY_SIZE(objects); ++i) { - unsigned int size = objects[i].size; - unsigned int expected_gtt = objects[i].gtt; - unsigned int offset = objects[i].offset; - unsigned int flags = PIN_USER; - - for (single = 0; single <= 1; single++) { - obj = fake_huge_pages_object(i915, size, !!single); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - err = i915_gem_object_pin_pages(obj); - if (err) - goto out_object_put; - - /* - * Disable 2M pages -- We only want to use 64K/4K pages - * for this test. - */ - obj->mm.page_sizes.sg &= ~I915_GTT_PAGE_SIZE_2M; - - vma = i915_vma_instance(obj, &ppgtt->vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto out_object_unpin; - } - - if (offset) - flags |= PIN_OFFSET_FIXED | offset; - - err = i915_vma_pin(vma, 0, 0, flags); - if (err) - goto out_vma_close; - - err = igt_check_page_sizes(vma); - if (err) - goto out_vma_unpin; - - if (!offset && vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) { - if (!IS_ALIGNED(vma->node.start, - I915_GTT_PAGE_SIZE_2M)) { - pr_err("node.start(%llx) not aligned to 2M\n", - vma->node.start); - err = -EINVAL; - goto out_vma_unpin; - } - - if (!IS_ALIGNED(vma->node.size, - I915_GTT_PAGE_SIZE_2M)) { - pr_err("node.size(%llx) not aligned to 2M\n", - vma->node.size); - err = -EINVAL; - goto out_vma_unpin; - } - } - - if (vma->page_sizes.gtt != expected_gtt) { - pr_err("gtt=%u, expected=%u, i=%d, single=%s\n", - vma->page_sizes.gtt, expected_gtt, i, - yesno(!!single)); - err = -EINVAL; - goto out_vma_unpin; - } - - i915_vma_unpin(vma); - i915_vma_close(vma); - - i915_gem_object_unpin_pages(obj); - __i915_gem_object_put_pages(obj, I915_MM_NORMAL); - i915_gem_object_put(obj); - } - } - - return 0; - -out_vma_unpin: - i915_vma_unpin(vma); -out_vma_close: - i915_vma_close(vma); -out_object_unpin: - i915_gem_object_unpin_pages(obj); -out_object_put: - i915_gem_object_put(obj); - - return err; -} - -static struct i915_vma * -gpu_write_dw(struct i915_vma *vma, u64 offset, u32 val) -{ - struct drm_i915_private *i915 = vma->vm->i915; - const int gen = INTEL_GEN(i915); - unsigned int count = vma->size >> PAGE_SHIFT; - struct drm_i915_gem_object *obj; - struct i915_vma *batch; - unsigned int size; - u32 *cmd; - int n; - int err; - - size = (1 + 4 * count) * sizeof(u32); - size = round_up(size, PAGE_SIZE); - obj = i915_gem_object_create_internal(i915, size); - if (IS_ERR(obj)) - return ERR_CAST(obj); - - cmd = i915_gem_object_pin_map(obj, I915_MAP_WC); - if (IS_ERR(cmd)) { - err = PTR_ERR(cmd); - goto err; - } - - offset += vma->node.start; - - for (n = 0; n < count; n++) { - if (gen >= 8) { - *cmd++ = MI_STORE_DWORD_IMM_GEN4; - *cmd++ = lower_32_bits(offset); - *cmd++ = upper_32_bits(offset); - *cmd++ = val; - } else if (gen >= 4) { - *cmd++ = MI_STORE_DWORD_IMM_GEN4 | - (gen < 6 ? MI_USE_GGTT : 0); - *cmd++ = 0; - *cmd++ = offset; - *cmd++ = val; - } else { - *cmd++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; - *cmd++ = offset; - *cmd++ = val; - } - - offset += PAGE_SIZE; - } - - *cmd = MI_BATCH_BUFFER_END; - i915_gem_chipset_flush(i915); - - i915_gem_object_unpin_map(obj); - - batch = i915_vma_instance(obj, vma->vm, NULL); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto err; - } - - err = i915_vma_pin(batch, 0, 0, PIN_USER); - if (err) - goto err; - - return batch; - -err: - i915_gem_object_put(obj); - - return ERR_PTR(err); -} - -static int gpu_write(struct i915_vma *vma, - struct i915_gem_context *ctx, - struct intel_engine_cs *engine, - u32 dword, - u32 value) -{ - struct i915_request *rq; - struct i915_vma *batch; - int err; - - GEM_BUG_ON(!intel_engine_can_store_dword(engine)); - - err = i915_gem_object_set_to_gtt_domain(vma->obj, true); - if (err) - return err; - - batch = gpu_write_dw(vma, dword * sizeof(u32), value); - if (IS_ERR(batch)) - return PTR_ERR(batch); - - rq = i915_request_alloc(engine, ctx); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_batch; - } - - err = i915_vma_move_to_active(batch, rq, 0); - if (err) - goto err_request; - - i915_gem_object_set_active_reference(batch->obj); - - err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); - if (err) - goto err_request; - - err = engine->emit_bb_start(rq, - batch->node.start, batch->node.size, - 0); -err_request: - if (err) - i915_request_skip(rq, err); - i915_request_add(rq); -err_batch: - i915_vma_unpin(batch); - i915_vma_close(batch); - - return err; -} - -static int cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val) -{ - unsigned int needs_flush; - unsigned long n; - int err; - - err = i915_gem_obj_prepare_shmem_read(obj, &needs_flush); - if (err) - return err; - - for (n = 0; n < obj->base.size >> PAGE_SHIFT; ++n) { - u32 *ptr = kmap_atomic(i915_gem_object_get_page(obj, n)); - - if (needs_flush & CLFLUSH_BEFORE) - drm_clflush_virt_range(ptr, PAGE_SIZE); - - if (ptr[dword] != val) { - pr_err("n=%lu ptr[%u]=%u, val=%u\n", - n, dword, ptr[dword], val); - kunmap_atomic(ptr); - err = -EINVAL; - break; - } - - kunmap_atomic(ptr); - } - - i915_gem_obj_finish_shmem_access(obj); - - return err; -} - -static int __igt_write_huge(struct i915_gem_context *ctx, - struct intel_engine_cs *engine, - struct drm_i915_gem_object *obj, - u64 size, u64 offset, - u32 dword, u32 val) -{ - struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_address_space *vm = - ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm; - unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; - struct i915_vma *vma; - int err; - - vma = i915_vma_instance(obj, vm, NULL); - if (IS_ERR(vma)) - return PTR_ERR(vma); - - err = i915_vma_unbind(vma); - if (err) - goto out_vma_close; - - err = i915_vma_pin(vma, size, 0, flags | offset); - if (err) { - /* - * The ggtt may have some pages reserved so - * refrain from erroring out. - */ - if (err == -ENOSPC && i915_is_ggtt(vm)) - err = 0; - - goto out_vma_close; - } - - err = igt_check_page_sizes(vma); - if (err) - goto out_vma_unpin; - - err = gpu_write(vma, ctx, engine, dword, val); - if (err) { - pr_err("gpu-write failed at offset=%llx\n", offset); - goto out_vma_unpin; - } - - err = cpu_check(obj, dword, val); - if (err) { - pr_err("cpu-check failed at offset=%llx\n", offset); - goto out_vma_unpin; - } - -out_vma_unpin: - i915_vma_unpin(vma); -out_vma_close: - i915_vma_destroy(vma); - - return err; -} - -static int igt_write_huge(struct i915_gem_context *ctx, - struct drm_i915_gem_object *obj) -{ - struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_address_space *vm = - ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm; - static struct intel_engine_cs *engines[I915_NUM_ENGINES]; - struct intel_engine_cs *engine; - I915_RND_STATE(prng); - IGT_TIMEOUT(end_time); - unsigned int max_page_size; - unsigned int id; - u64 max; - u64 num; - u64 size; - int *order; - int i, n; - int err = 0; - - GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); - - size = obj->base.size; - if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K) - size = round_up(size, I915_GTT_PAGE_SIZE_2M); - - max_page_size = rounddown_pow_of_two(obj->mm.page_sizes.sg); - max = div_u64((vm->total - size), max_page_size); - - n = 0; - for_each_engine(engine, i915, id) { - if (!intel_engine_can_store_dword(engine)) { - pr_info("store-dword-imm not supported on engine=%u\n", - id); - continue; - } - engines[n++] = engine; - } - - if (!n) - return 0; - - /* - * To keep things interesting when alternating between engines in our - * randomized order, lets also make feeding to the same engine a few - * times in succession a possibility by enlarging the permutation array. - */ - order = i915_random_order(n * I915_NUM_ENGINES, &prng); - if (!order) - return -ENOMEM; - - /* - * Try various offsets in an ascending/descending fashion until we - * timeout -- we want to avoid issues hidden by effectively always using - * offset = 0. - */ - i = 0; - for_each_prime_number_from(num, 0, max) { - u64 offset_low = num * max_page_size; - u64 offset_high = (max - num) * max_page_size; - u32 dword = offset_in_page(num) / 4; - - engine = engines[order[i] % n]; - i = (i + 1) % (n * I915_NUM_ENGINES); - - /* - * In order to utilize 64K pages we need to both pad the vma - * size and ensure the vma offset is at the start of the pt - * boundary, however to improve coverage we opt for testing both - * aligned and unaligned offsets. - */ - if (obj->mm.page_sizes.sg & I915_GTT_PAGE_SIZE_64K) - offset_low = round_down(offset_low, - I915_GTT_PAGE_SIZE_2M); - - err = __igt_write_huge(ctx, engine, obj, size, offset_low, - dword, num + 1); - if (err) - break; - - err = __igt_write_huge(ctx, engine, obj, size, offset_high, - dword, num + 1); - if (err) - break; - - if (igt_timeout(end_time, - "%s timed out on engine=%u, offset_low=%llx offset_high=%llx, max_page_size=%x\n", - __func__, engine->id, offset_low, offset_high, - max_page_size)) - break; - } - - kfree(order); - - return err; -} - -static int igt_ppgtt_exhaust_huge(void *arg) -{ - struct i915_gem_context *ctx = arg; - struct drm_i915_private *i915 = ctx->i915; - unsigned long supported = INTEL_INFO(i915)->page_sizes; - static unsigned int pages[ARRAY_SIZE(page_sizes)]; - struct drm_i915_gem_object *obj; - unsigned int size_mask; - unsigned int page_mask; - int n, i; - int err = -ENODEV; - - if (supported == I915_GTT_PAGE_SIZE_4K) - return 0; - - /* - * Sanity check creating objects with a varying mix of page sizes -- - * ensuring that our writes lands in the right place. - */ - - n = 0; - for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) - pages[n++] = BIT(i); - - for (size_mask = 2; size_mask < BIT(n); size_mask++) { - unsigned int size = 0; - - for (i = 0; i < n; i++) { - if (size_mask & BIT(i)) - size |= pages[i]; - } - - /* - * For our page mask we want to enumerate all the page-size - * combinations which will fit into our chosen object size. - */ - for (page_mask = 2; page_mask <= size_mask; page_mask++) { - unsigned int page_sizes = 0; - - for (i = 0; i < n; i++) { - if (page_mask & BIT(i)) - page_sizes |= pages[i]; - } - - /* - * Ensure that we can actually fill the given object - * with our chosen page mask. - */ - if (!IS_ALIGNED(size, BIT(__ffs(page_sizes)))) - continue; - - obj = huge_pages_object(i915, size, page_sizes); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto out_device; - } - - err = i915_gem_object_pin_pages(obj); - if (err) { - i915_gem_object_put(obj); - - if (err == -ENOMEM) { - pr_info("unable to get pages, size=%u, pages=%u\n", - size, page_sizes); - err = 0; - break; - } - - pr_err("pin_pages failed, size=%u, pages=%u\n", - size_mask, page_mask); - - goto out_device; - } - - /* Force the page-size for the gtt insertion */ - obj->mm.page_sizes.sg = page_sizes; - - err = igt_write_huge(ctx, obj); - if (err) { - pr_err("exhaust write-huge failed with size=%u\n", - size); - goto out_unpin; - } - - i915_gem_object_unpin_pages(obj); - __i915_gem_object_put_pages(obj, I915_MM_NORMAL); - i915_gem_object_put(obj); - } - } - - goto out_device; - -out_unpin: - i915_gem_object_unpin_pages(obj); - i915_gem_object_put(obj); -out_device: - mkwrite_device_info(i915)->page_sizes = supported; - - return err; -} - -static int igt_ppgtt_internal_huge(void *arg) -{ - struct i915_gem_context *ctx = arg; - struct drm_i915_private *i915 = ctx->i915; - struct drm_i915_gem_object *obj; - static const unsigned int sizes[] = { - SZ_64K, - SZ_128K, - SZ_256K, - SZ_512K, - SZ_1M, - SZ_2M, - }; - int i; - int err; - - /* - * Sanity check that the HW uses huge pages correctly through internal - * -- ensure that our writes land in the right place. - */ - - for (i = 0; i < ARRAY_SIZE(sizes); ++i) { - unsigned int size = sizes[i]; - - obj = i915_gem_object_create_internal(i915, size); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - err = i915_gem_object_pin_pages(obj); - if (err) - goto out_put; - - if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_64K) { - pr_info("internal unable to allocate huge-page(s) with size=%u\n", - size); - goto out_unpin; - } - - err = igt_write_huge(ctx, obj); - if (err) { - pr_err("internal write-huge failed with size=%u\n", - size); - goto out_unpin; - } - - i915_gem_object_unpin_pages(obj); - __i915_gem_object_put_pages(obj, I915_MM_NORMAL); - i915_gem_object_put(obj); - } - - return 0; - -out_unpin: - i915_gem_object_unpin_pages(obj); -out_put: - i915_gem_object_put(obj); - - return err; -} - -static inline bool igt_can_allocate_thp(struct drm_i915_private *i915) -{ - return i915->mm.gemfs && has_transparent_hugepage(); -} - -static int igt_ppgtt_gemfs_huge(void *arg) -{ - struct i915_gem_context *ctx = arg; - struct drm_i915_private *i915 = ctx->i915; - struct drm_i915_gem_object *obj; - static const unsigned int sizes[] = { - SZ_2M, - SZ_4M, - SZ_8M, - SZ_16M, - SZ_32M, - }; - int i; - int err; - - /* - * Sanity check that the HW uses huge pages correctly through gemfs -- - * ensure that our writes land in the right place. - */ - - if (!igt_can_allocate_thp(i915)) { - pr_info("missing THP support, skipping\n"); - return 0; - } - - for (i = 0; i < ARRAY_SIZE(sizes); ++i) { - unsigned int size = sizes[i]; - - obj = i915_gem_object_create(i915, size); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - err = i915_gem_object_pin_pages(obj); - if (err) - goto out_put; - - if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) { - pr_info("finishing test early, gemfs unable to allocate huge-page(s) with size=%u\n", - size); - goto out_unpin; - } - - err = igt_write_huge(ctx, obj); - if (err) { - pr_err("gemfs write-huge failed with size=%u\n", - size); - goto out_unpin; - } - - i915_gem_object_unpin_pages(obj); - __i915_gem_object_put_pages(obj, I915_MM_NORMAL); - i915_gem_object_put(obj); - } - - return 0; - -out_unpin: - i915_gem_object_unpin_pages(obj); -out_put: - i915_gem_object_put(obj); - - return err; -} - -static int igt_ppgtt_pin_update(void *arg) -{ - struct i915_gem_context *ctx = arg; - struct drm_i915_private *dev_priv = ctx->i915; - unsigned long supported = INTEL_INFO(dev_priv)->page_sizes; - struct i915_hw_ppgtt *ppgtt = ctx->ppgtt; - struct drm_i915_gem_object *obj; - struct i915_vma *vma; - unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; - int first, last; - int err; - - /* - * Make sure there's no funny business when doing a PIN_UPDATE -- in the - * past we had a subtle issue with being able to incorrectly do multiple - * alloc va ranges on the same object when doing a PIN_UPDATE, which - * resulted in some pretty nasty bugs, though only when using - * huge-gtt-pages. - */ - - if (!ppgtt || !i915_vm_is_4lvl(&ppgtt->vm)) { - pr_info("48b PPGTT not supported, skipping\n"); - return 0; - } - - first = ilog2(I915_GTT_PAGE_SIZE_64K); - last = ilog2(I915_GTT_PAGE_SIZE_2M); - - for_each_set_bit_from(first, &supported, last + 1) { - unsigned int page_size = BIT(first); - - obj = i915_gem_object_create_internal(dev_priv, page_size); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - vma = i915_vma_instance(obj, &ppgtt->vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto out_put; - } - - err = i915_vma_pin(vma, SZ_2M, 0, flags); - if (err) - goto out_close; - - if (vma->page_sizes.sg < page_size) { - pr_info("Unable to allocate page-size %x, finishing test early\n", - page_size); - goto out_unpin; - } - - err = igt_check_page_sizes(vma); - if (err) - goto out_unpin; - - if (vma->page_sizes.gtt != page_size) { - dma_addr_t addr = i915_gem_object_get_dma_address(obj, 0); - - /* - * The only valid reason for this to ever fail would be - * if the dma-mapper screwed us over when we did the - * dma_map_sg(), since it has the final say over the dma - * address. - */ - if (IS_ALIGNED(addr, page_size)) { - pr_err("page_sizes.gtt=%u, expected=%u\n", - vma->page_sizes.gtt, page_size); - err = -EINVAL; - } else { - pr_info("dma address misaligned, finishing test early\n"); - } - - goto out_unpin; - } - - err = i915_vma_bind(vma, I915_CACHE_NONE, PIN_UPDATE); - if (err) - goto out_unpin; - - i915_vma_unpin(vma); - i915_vma_close(vma); - - i915_gem_object_put(obj); - } - - obj = i915_gem_object_create_internal(dev_priv, PAGE_SIZE); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - vma = i915_vma_instance(obj, &ppgtt->vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto out_put; - } - - err = i915_vma_pin(vma, 0, 0, flags); - if (err) - goto out_close; - - /* - * Make sure we don't end up with something like where the pde is still - * pointing to the 2M page, and the pt we just filled-in is dangling -- - * we can check this by writing to the first page where it would then - * land in the now stale 2M page. - */ - - err = gpu_write(vma, ctx, dev_priv->engine[RCS0], 0, 0xdeadbeaf); - if (err) - goto out_unpin; - - err = cpu_check(obj, 0, 0xdeadbeaf); - -out_unpin: - i915_vma_unpin(vma); -out_close: - i915_vma_close(vma); -out_put: - i915_gem_object_put(obj); - - return err; -} - -static int igt_tmpfs_fallback(void *arg) -{ - struct i915_gem_context *ctx = arg; - struct drm_i915_private *i915 = ctx->i915; - struct vfsmount *gemfs = i915->mm.gemfs; - struct i915_address_space *vm = - ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm; - struct drm_i915_gem_object *obj; - struct i915_vma *vma; - u32 *vaddr; - int err = 0; - - /* - * Make sure that we don't burst into a ball of flames upon falling back - * to tmpfs, which we rely on if on the off-chance we encouter a failure - * when setting up gemfs. - */ - - i915->mm.gemfs = NULL; - - obj = i915_gem_object_create(i915, PAGE_SIZE); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto out_restore; - } - - vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); - if (IS_ERR(vaddr)) { - err = PTR_ERR(vaddr); - goto out_put; - } - *vaddr = 0xdeadbeaf; - - __i915_gem_object_flush_map(obj, 0, 64); - i915_gem_object_unpin_map(obj); - - vma = i915_vma_instance(obj, vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto out_put; - } - - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) - goto out_close; - - err = igt_check_page_sizes(vma); - - i915_vma_unpin(vma); -out_close: - i915_vma_close(vma); -out_put: - i915_gem_object_put(obj); -out_restore: - i915->mm.gemfs = gemfs; - - return err; -} - -static int igt_shrink_thp(void *arg) -{ - struct i915_gem_context *ctx = arg; - struct drm_i915_private *i915 = ctx->i915; - struct i915_address_space *vm = - ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm; - struct drm_i915_gem_object *obj; - struct i915_vma *vma; - unsigned int flags = PIN_USER; - int err; - - /* - * Sanity check shrinking huge-paged object -- make sure nothing blows - * up. - */ - - if (!igt_can_allocate_thp(i915)) { - pr_info("missing THP support, skipping\n"); - return 0; - } - - obj = i915_gem_object_create(i915, SZ_2M); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - vma = i915_vma_instance(obj, vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto out_put; - } - - err = i915_vma_pin(vma, 0, 0, flags); - if (err) - goto out_close; - - if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) { - pr_info("failed to allocate THP, finishing test early\n"); - goto out_unpin; - } - - err = igt_check_page_sizes(vma); - if (err) - goto out_unpin; - - err = gpu_write(vma, ctx, i915->engine[RCS0], 0, 0xdeadbeaf); - if (err) - goto out_unpin; - - i915_vma_unpin(vma); - - /* - * Now that the pages are *unpinned* shrink-all should invoke - * shmem to truncate our pages. - */ - i915_gem_shrink_all(i915); - if (i915_gem_object_has_pages(obj)) { - pr_err("shrink-all didn't truncate the pages\n"); - err = -EINVAL; - goto out_close; - } - - if (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys) { - pr_err("residual page-size bits left\n"); - err = -EINVAL; - goto out_close; - } - - err = i915_vma_pin(vma, 0, 0, flags); - if (err) - goto out_close; - - err = cpu_check(obj, 0, 0xdeadbeaf); - -out_unpin: - i915_vma_unpin(vma); -out_close: - i915_vma_close(vma); -out_put: - i915_gem_object_put(obj); - - return err; -} - -int i915_gem_huge_page_mock_selftests(void) -{ - static const struct i915_subtest tests[] = { - SUBTEST(igt_mock_exhaust_device_supported_pages), - SUBTEST(igt_mock_ppgtt_misaligned_dma), - SUBTEST(igt_mock_ppgtt_huge_fill), - SUBTEST(igt_mock_ppgtt_64K), - }; - struct drm_i915_private *dev_priv; - struct i915_hw_ppgtt *ppgtt; - int err; - - dev_priv = mock_gem_device(); - if (!dev_priv) - return -ENOMEM; - - /* Pretend to be a device which supports the 48b PPGTT */ - mkwrite_device_info(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL; - mkwrite_device_info(dev_priv)->ppgtt_size = 48; - - mutex_lock(&dev_priv->drm.struct_mutex); - ppgtt = i915_ppgtt_create(dev_priv); - if (IS_ERR(ppgtt)) { - err = PTR_ERR(ppgtt); - goto out_unlock; - } - - if (!i915_vm_is_4lvl(&ppgtt->vm)) { - pr_err("failed to create 48b PPGTT\n"); - err = -EINVAL; - goto out_close; - } - - /* If we were ever hit this then it's time to mock the 64K scratch */ - if (!i915_vm_has_scratch_64K(&ppgtt->vm)) { - pr_err("PPGTT missing 64K scratch page\n"); - err = -EINVAL; - goto out_close; - } - - err = i915_subtests(tests, ppgtt); - -out_close: - i915_ppgtt_put(ppgtt); - -out_unlock: - mutex_unlock(&dev_priv->drm.struct_mutex); - drm_dev_put(&dev_priv->drm); - - return err; -} - -int i915_gem_huge_page_live_selftests(struct drm_i915_private *dev_priv) -{ - static const struct i915_subtest tests[] = { - SUBTEST(igt_shrink_thp), - SUBTEST(igt_ppgtt_pin_update), - SUBTEST(igt_tmpfs_fallback), - SUBTEST(igt_ppgtt_exhaust_huge), - SUBTEST(igt_ppgtt_gemfs_huge), - SUBTEST(igt_ppgtt_internal_huge), - }; - struct drm_file *file; - struct i915_gem_context *ctx; - intel_wakeref_t wakeref; - int err; - - if (!HAS_PPGTT(dev_priv)) { - pr_info("PPGTT not supported, skipping live-selftests\n"); - return 0; - } - - if (i915_terminally_wedged(dev_priv)) - return 0; - - file = mock_file(dev_priv); - if (IS_ERR(file)) - return PTR_ERR(file); - - mutex_lock(&dev_priv->drm.struct_mutex); - wakeref = intel_runtime_pm_get(dev_priv); - - ctx = live_context(dev_priv, file); - if (IS_ERR(ctx)) { - err = PTR_ERR(ctx); - goto out_unlock; - } - - if (ctx->ppgtt) - ctx->ppgtt->vm.scrub_64K = true; - - err = i915_subtests(tests, ctx); - -out_unlock: - intel_runtime_pm_put(dev_priv, wakeref); - mutex_unlock(&dev_priv->drm.struct_mutex); - - mock_file_free(dev_priv, file); - - return err; -} diff --git a/drivers/gpu/drm/i915/selftests/i915_active.c b/drivers/gpu/drm/i915/selftests/i915_active.c index 27d8f853111b..c0b3537a5fa6 100644 --- a/drivers/gpu/drm/i915/selftests/i915_active.c +++ b/drivers/gpu/drm/i915/selftests/i915_active.c @@ -4,7 +4,9 @@ * Copyright © 2018 Intel Corporation */ -#include "../i915_selftest.h" +#include "gem/i915_gem_pm.h" + +#include "i915_selftest.h" #include "igt_flush_test.h" #include "lib_sw_fence.h" @@ -46,7 +48,7 @@ static int __live_active_setup(struct drm_i915_private *i915, for_each_engine(engine, i915, id) { struct i915_request *rq; - rq = i915_request_alloc(engine, i915->kernel_context); + rq = i915_request_create(engine->kernel_context); if (IS_ERR(rq)) { err = PTR_ERR(rq); break; @@ -95,7 +97,7 @@ static int live_active_wait(void *arg) /* Check that we get a callback when requests retire upon waiting */ mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); err = __live_active_setup(i915, &active); @@ -109,7 +111,7 @@ static int live_active_wait(void *arg) if (igt_flush_test(i915, I915_WAIT_LOCKED)) err = -EIO; - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; } @@ -124,7 +126,7 @@ static int live_active_retire(void *arg) /* Check that we get a callback when requests are indirectly retired */ mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); err = __live_active_setup(i915, &active); @@ -138,7 +140,7 @@ static int live_active_retire(void *arg) } i915_active_fini(&active.base); - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; } diff --git a/drivers/gpu/drm/i915/selftests/i915_gem.c b/drivers/gpu/drm/i915/selftests/i915_gem.c index 6fd70d326468..c6a01a6e87f1 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem.c @@ -6,36 +6,31 @@ #include <linux/random.h> -#include "../i915_selftest.h" +#include "gem/selftests/igt_gem_utils.h" +#include "gem/selftests/mock_context.h" + +#include "i915_selftest.h" -#include "mock_context.h" #include "igt_flush_test.h" +#include "mock_drm.h" static int switch_to_context(struct drm_i915_private *i915, struct i915_gem_context *ctx) { struct intel_engine_cs *engine; enum intel_engine_id id; - intel_wakeref_t wakeref; - int err = 0; - - wakeref = intel_runtime_pm_get(i915); for_each_engine(engine, i915, id) { struct i915_request *rq; - rq = i915_request_alloc(engine, ctx); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - break; - } + rq = igt_request_alloc(ctx, engine); + if (IS_ERR(rq)) + return PTR_ERR(rq); i915_request_add(rq); } - intel_runtime_pm_put(i915, wakeref); - - return err; + return 0; } static void trash_stolen(struct drm_i915_private *i915) @@ -68,7 +63,7 @@ static void simulate_hibernate(struct drm_i915_private *i915) { intel_wakeref_t wakeref; - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); /* * As a final sting in the tail, invalidate stolen. Under a real S4, @@ -79,7 +74,7 @@ static void simulate_hibernate(struct drm_i915_private *i915) */ trash_stolen(i915); - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); } static int pm_prepare(struct drm_i915_private *i915) @@ -93,7 +88,7 @@ static void pm_suspend(struct drm_i915_private *i915) { intel_wakeref_t wakeref; - with_intel_runtime_pm(i915, wakeref) { + with_intel_runtime_pm(&i915->runtime_pm, wakeref) { i915_gem_suspend_gtt_mappings(i915); i915_gem_suspend_late(i915); } @@ -103,7 +98,7 @@ static void pm_hibernate(struct drm_i915_private *i915) { intel_wakeref_t wakeref; - with_intel_runtime_pm(i915, wakeref) { + with_intel_runtime_pm(&i915->runtime_pm, wakeref) { i915_gem_suspend_gtt_mappings(i915); i915_gem_freeze(i915); @@ -119,8 +114,8 @@ static void pm_resume(struct drm_i915_private *i915) * Both suspend and hibernate follow the same wakeup path and assume * that runtime-pm just works. */ - with_intel_runtime_pm(i915, wakeref) { - intel_engines_sanitize(i915, false); + with_intel_runtime_pm(&i915->runtime_pm, wakeref) { + intel_gt_sanitize(i915, false); i915_gem_sanitize(i915); i915_gem_resume(i915); } diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c deleted file mode 100644 index e43630b40fce..000000000000 --- a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c +++ /dev/null @@ -1,397 +0,0 @@ -/* - * Copyright © 2017 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#include <linux/prime_numbers.h> - -#include "../i915_selftest.h" -#include "i915_random.h" - -static int cpu_set(struct drm_i915_gem_object *obj, - unsigned long offset, - u32 v) -{ - unsigned int needs_clflush; - struct page *page; - void *map; - u32 *cpu; - int err; - - err = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush); - if (err) - return err; - - page = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT); - map = kmap_atomic(page); - cpu = map + offset_in_page(offset); - - if (needs_clflush & CLFLUSH_BEFORE) - drm_clflush_virt_range(cpu, sizeof(*cpu)); - - *cpu = v; - - if (needs_clflush & CLFLUSH_AFTER) - drm_clflush_virt_range(cpu, sizeof(*cpu)); - - kunmap_atomic(map); - i915_gem_obj_finish_shmem_access(obj); - - return 0; -} - -static int cpu_get(struct drm_i915_gem_object *obj, - unsigned long offset, - u32 *v) -{ - unsigned int needs_clflush; - struct page *page; - void *map; - u32 *cpu; - int err; - - err = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush); - if (err) - return err; - - page = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT); - map = kmap_atomic(page); - cpu = map + offset_in_page(offset); - - if (needs_clflush & CLFLUSH_BEFORE) - drm_clflush_virt_range(cpu, sizeof(*cpu)); - - *v = *cpu; - - kunmap_atomic(map); - i915_gem_obj_finish_shmem_access(obj); - - return 0; -} - -static int gtt_set(struct drm_i915_gem_object *obj, - unsigned long offset, - u32 v) -{ - struct i915_vma *vma; - u32 __iomem *map; - int err; - - err = i915_gem_object_set_to_gtt_domain(obj, true); - if (err) - return err; - - vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE); - if (IS_ERR(vma)) - return PTR_ERR(vma); - - map = i915_vma_pin_iomap(vma); - i915_vma_unpin(vma); - if (IS_ERR(map)) - return PTR_ERR(map); - - iowrite32(v, &map[offset / sizeof(*map)]); - i915_vma_unpin_iomap(vma); - - return 0; -} - -static int gtt_get(struct drm_i915_gem_object *obj, - unsigned long offset, - u32 *v) -{ - struct i915_vma *vma; - u32 __iomem *map; - int err; - - err = i915_gem_object_set_to_gtt_domain(obj, false); - if (err) - return err; - - vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE); - if (IS_ERR(vma)) - return PTR_ERR(vma); - - map = i915_vma_pin_iomap(vma); - i915_vma_unpin(vma); - if (IS_ERR(map)) - return PTR_ERR(map); - - *v = ioread32(&map[offset / sizeof(*map)]); - i915_vma_unpin_iomap(vma); - - return 0; -} - -static int wc_set(struct drm_i915_gem_object *obj, - unsigned long offset, - u32 v) -{ - u32 *map; - int err; - - err = i915_gem_object_set_to_wc_domain(obj, true); - if (err) - return err; - - map = i915_gem_object_pin_map(obj, I915_MAP_WC); - if (IS_ERR(map)) - return PTR_ERR(map); - - map[offset / sizeof(*map)] = v; - i915_gem_object_unpin_map(obj); - - return 0; -} - -static int wc_get(struct drm_i915_gem_object *obj, - unsigned long offset, - u32 *v) -{ - u32 *map; - int err; - - err = i915_gem_object_set_to_wc_domain(obj, false); - if (err) - return err; - - map = i915_gem_object_pin_map(obj, I915_MAP_WC); - if (IS_ERR(map)) - return PTR_ERR(map); - - *v = map[offset / sizeof(*map)]; - i915_gem_object_unpin_map(obj); - - return 0; -} - -static int gpu_set(struct drm_i915_gem_object *obj, - unsigned long offset, - u32 v) -{ - struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_request *rq; - struct i915_vma *vma; - u32 *cs; - int err; - - err = i915_gem_object_set_to_gtt_domain(obj, true); - if (err) - return err; - - vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0); - if (IS_ERR(vma)) - return PTR_ERR(vma); - - rq = i915_request_alloc(i915->engine[RCS0], i915->kernel_context); - if (IS_ERR(rq)) { - i915_vma_unpin(vma); - return PTR_ERR(rq); - } - - cs = intel_ring_begin(rq, 4); - if (IS_ERR(cs)) { - i915_request_add(rq); - i915_vma_unpin(vma); - return PTR_ERR(cs); - } - - if (INTEL_GEN(i915) >= 8) { - *cs++ = MI_STORE_DWORD_IMM_GEN4 | 1 << 22; - *cs++ = lower_32_bits(i915_ggtt_offset(vma) + offset); - *cs++ = upper_32_bits(i915_ggtt_offset(vma) + offset); - *cs++ = v; - } else if (INTEL_GEN(i915) >= 4) { - *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; - *cs++ = 0; - *cs++ = i915_ggtt_offset(vma) + offset; - *cs++ = v; - } else { - *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; - *cs++ = i915_ggtt_offset(vma) + offset; - *cs++ = v; - *cs++ = MI_NOOP; - } - intel_ring_advance(rq, cs); - - err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); - i915_vma_unpin(vma); - - i915_request_add(rq); - - return err; -} - -static bool always_valid(struct drm_i915_private *i915) -{ - return true; -} - -static bool needs_fence_registers(struct drm_i915_private *i915) -{ - return !i915_terminally_wedged(i915); -} - -static bool needs_mi_store_dword(struct drm_i915_private *i915) -{ - if (i915_terminally_wedged(i915)) - return false; - - return intel_engine_can_store_dword(i915->engine[RCS0]); -} - -static const struct igt_coherency_mode { - const char *name; - int (*set)(struct drm_i915_gem_object *, unsigned long offset, u32 v); - int (*get)(struct drm_i915_gem_object *, unsigned long offset, u32 *v); - bool (*valid)(struct drm_i915_private *i915); -} igt_coherency_mode[] = { - { "cpu", cpu_set, cpu_get, always_valid }, - { "gtt", gtt_set, gtt_get, needs_fence_registers }, - { "wc", wc_set, wc_get, always_valid }, - { "gpu", gpu_set, NULL, needs_mi_store_dword }, - { }, -}; - -static int igt_gem_coherency(void *arg) -{ - const unsigned int ncachelines = PAGE_SIZE/64; - I915_RND_STATE(prng); - struct drm_i915_private *i915 = arg; - const struct igt_coherency_mode *read, *write, *over; - struct drm_i915_gem_object *obj; - intel_wakeref_t wakeref; - unsigned long count, n; - u32 *offsets, *values; - int err = 0; - - /* We repeatedly write, overwrite and read from a sequence of - * cachelines in order to try and detect incoherency (unflushed writes - * from either the CPU or GPU). Each setter/getter uses our cache - * domain API which should prevent incoherency. - */ - - offsets = kmalloc_array(ncachelines, 2*sizeof(u32), GFP_KERNEL); - if (!offsets) - return -ENOMEM; - for (count = 0; count < ncachelines; count++) - offsets[count] = count * 64 + 4 * (count % 16); - - values = offsets + ncachelines; - - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); - for (over = igt_coherency_mode; over->name; over++) { - if (!over->set) - continue; - - if (!over->valid(i915)) - continue; - - for (write = igt_coherency_mode; write->name; write++) { - if (!write->set) - continue; - - if (!write->valid(i915)) - continue; - - for (read = igt_coherency_mode; read->name; read++) { - if (!read->get) - continue; - - if (!read->valid(i915)) - continue; - - for_each_prime_number_from(count, 1, ncachelines) { - obj = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto unlock; - } - - i915_random_reorder(offsets, ncachelines, &prng); - for (n = 0; n < count; n++) - values[n] = prandom_u32_state(&prng); - - for (n = 0; n < count; n++) { - err = over->set(obj, offsets[n], ~values[n]); - if (err) { - pr_err("Failed to set stale value[%ld/%ld] in object using %s, err=%d\n", - n, count, over->name, err); - goto put_object; - } - } - - for (n = 0; n < count; n++) { - err = write->set(obj, offsets[n], values[n]); - if (err) { - pr_err("Failed to set value[%ld/%ld] in object using %s, err=%d\n", - n, count, write->name, err); - goto put_object; - } - } - - for (n = 0; n < count; n++) { - u32 found; - - err = read->get(obj, offsets[n], &found); - if (err) { - pr_err("Failed to get value[%ld/%ld] in object using %s, err=%d\n", - n, count, read->name, err); - goto put_object; - } - - if (found != values[n]) { - pr_err("Value[%ld/%ld] mismatch, (overwrite with %s) wrote [%s] %x read [%s] %x (inverse %x), at offset %x\n", - n, count, over->name, - write->name, values[n], - read->name, found, - ~values[n], offsets[n]); - err = -EINVAL; - goto put_object; - } - } - - __i915_gem_object_release_unless_active(obj); - } - } - } - } -unlock: - intel_runtime_pm_put(i915, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - kfree(offsets); - return err; - -put_object: - __i915_gem_object_release_unless_active(obj); - goto unlock; -} - -int i915_gem_coherency_live_selftests(struct drm_i915_private *i915) -{ - static const struct i915_subtest tests[] = { - SUBTEST(igt_gem_coherency), - }; - - return i915_subtests(tests, i915); -} diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/selftests/i915_gem_context.c deleted file mode 100644 index 4e1b6efc6b22..000000000000 --- a/drivers/gpu/drm/i915/selftests/i915_gem_context.c +++ /dev/null @@ -1,1859 +0,0 @@ -/* - * Copyright © 2017 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#include <linux/prime_numbers.h> - -#include "../i915_reset.h" -#include "../i915_selftest.h" -#include "i915_random.h" -#include "igt_flush_test.h" -#include "igt_live_test.h" -#include "igt_reset.h" -#include "igt_spinner.h" - -#include "mock_drm.h" -#include "mock_gem_device.h" -#include "huge_gem_object.h" - -#define DW_PER_PAGE (PAGE_SIZE / sizeof(u32)) - -static int live_nop_switch(void *arg) -{ - const unsigned int nctx = 1024; - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *engine; - struct i915_gem_context **ctx; - enum intel_engine_id id; - intel_wakeref_t wakeref; - struct igt_live_test t; - struct drm_file *file; - unsigned long n; - int err = -ENODEV; - - /* - * Create as many contexts as we can feasibly get away with - * and check we can switch between them rapidly. - * - * Serves as very simple stress test for submission and HW switching - * between contexts. - */ - - if (!DRIVER_CAPS(i915)->has_logical_contexts) - return 0; - - file = mock_file(i915); - if (IS_ERR(file)) - return PTR_ERR(file); - - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); - - ctx = kcalloc(nctx, sizeof(*ctx), GFP_KERNEL); - if (!ctx) { - err = -ENOMEM; - goto out_unlock; - } - - for (n = 0; n < nctx; n++) { - ctx[n] = live_context(i915, file); - if (IS_ERR(ctx[n])) { - err = PTR_ERR(ctx[n]); - goto out_unlock; - } - } - - for_each_engine(engine, i915, id) { - struct i915_request *rq; - unsigned long end_time, prime; - ktime_t times[2] = {}; - - times[0] = ktime_get_raw(); - for (n = 0; n < nctx; n++) { - rq = i915_request_alloc(engine, ctx[n]); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto out_unlock; - } - i915_request_add(rq); - } - if (i915_request_wait(rq, - I915_WAIT_LOCKED, - HZ / 5) < 0) { - pr_err("Failed to populated %d contexts\n", nctx); - i915_gem_set_wedged(i915); - err = -EIO; - goto out_unlock; - } - - times[1] = ktime_get_raw(); - - pr_info("Populated %d contexts on %s in %lluns\n", - nctx, engine->name, ktime_to_ns(times[1] - times[0])); - - err = igt_live_test_begin(&t, i915, __func__, engine->name); - if (err) - goto out_unlock; - - end_time = jiffies + i915_selftest.timeout_jiffies; - for_each_prime_number_from(prime, 2, 8192) { - times[1] = ktime_get_raw(); - - for (n = 0; n < prime; n++) { - rq = i915_request_alloc(engine, ctx[n % nctx]); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto out_unlock; - } - - /* - * This space is left intentionally blank. - * - * We do not actually want to perform any - * action with this request, we just want - * to measure the latency in allocation - * and submission of our breadcrumbs - - * ensuring that the bare request is sufficient - * for the system to work (i.e. proper HEAD - * tracking of the rings, interrupt handling, - * etc). It also gives us the lowest bounds - * for latency. - */ - - i915_request_add(rq); - } - if (i915_request_wait(rq, - I915_WAIT_LOCKED, - HZ / 5) < 0) { - pr_err("Switching between %ld contexts timed out\n", - prime); - i915_gem_set_wedged(i915); - break; - } - - times[1] = ktime_sub(ktime_get_raw(), times[1]); - if (prime == 2) - times[0] = times[1]; - - if (__igt_timeout(end_time, NULL)) - break; - } - - err = igt_live_test_end(&t); - if (err) - goto out_unlock; - - pr_info("Switch latencies on %s: 1 = %lluns, %lu = %lluns\n", - engine->name, - ktime_to_ns(times[0]), - prime - 1, div64_u64(ktime_to_ns(times[1]), prime - 1)); - } - -out_unlock: - intel_runtime_pm_put(i915, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - mock_file_free(i915, file); - return err; -} - -static struct i915_vma * -gpu_fill_dw(struct i915_vma *vma, u64 offset, unsigned long count, u32 value) -{ - struct drm_i915_gem_object *obj; - const int gen = INTEL_GEN(vma->vm->i915); - unsigned long n, size; - u32 *cmd; - int err; - - size = (4 * count + 1) * sizeof(u32); - size = round_up(size, PAGE_SIZE); - obj = i915_gem_object_create_internal(vma->vm->i915, size); - if (IS_ERR(obj)) - return ERR_CAST(obj); - - cmd = i915_gem_object_pin_map(obj, I915_MAP_WB); - if (IS_ERR(cmd)) { - err = PTR_ERR(cmd); - goto err; - } - - GEM_BUG_ON(offset + (count - 1) * PAGE_SIZE > vma->node.size); - offset += vma->node.start; - - for (n = 0; n < count; n++) { - if (gen >= 8) { - *cmd++ = MI_STORE_DWORD_IMM_GEN4; - *cmd++ = lower_32_bits(offset); - *cmd++ = upper_32_bits(offset); - *cmd++ = value; - } else if (gen >= 4) { - *cmd++ = MI_STORE_DWORD_IMM_GEN4 | - (gen < 6 ? MI_USE_GGTT : 0); - *cmd++ = 0; - *cmd++ = offset; - *cmd++ = value; - } else { - *cmd++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; - *cmd++ = offset; - *cmd++ = value; - } - offset += PAGE_SIZE; - } - *cmd = MI_BATCH_BUFFER_END; - i915_gem_object_flush_map(obj); - i915_gem_object_unpin_map(obj); - - err = i915_gem_object_set_to_gtt_domain(obj, false); - if (err) - goto err; - - vma = i915_vma_instance(obj, vma->vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto err; - } - - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) - goto err; - - return vma; - -err: - i915_gem_object_put(obj); - return ERR_PTR(err); -} - -static unsigned long real_page_count(struct drm_i915_gem_object *obj) -{ - return huge_gem_object_phys_size(obj) >> PAGE_SHIFT; -} - -static unsigned long fake_page_count(struct drm_i915_gem_object *obj) -{ - return huge_gem_object_dma_size(obj) >> PAGE_SHIFT; -} - -static int gpu_fill(struct drm_i915_gem_object *obj, - struct i915_gem_context *ctx, - struct intel_engine_cs *engine, - unsigned int dw) -{ - struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_address_space *vm = - ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm; - struct i915_request *rq; - struct i915_vma *vma; - struct i915_vma *batch; - unsigned int flags; - int err; - - GEM_BUG_ON(obj->base.size > vm->total); - GEM_BUG_ON(!intel_engine_can_store_dword(engine)); - - vma = i915_vma_instance(obj, vm, NULL); - if (IS_ERR(vma)) - return PTR_ERR(vma); - - err = i915_gem_object_set_to_gtt_domain(obj, false); - if (err) - return err; - - err = i915_vma_pin(vma, 0, 0, PIN_HIGH | PIN_USER); - if (err) - return err; - - /* Within the GTT the huge objects maps every page onto - * its 1024 real pages (using phys_pfn = dma_pfn % 1024). - * We set the nth dword within the page using the nth - * mapping via the GTT - this should exercise the GTT mapping - * whilst checking that each context provides a unique view - * into the object. - */ - batch = gpu_fill_dw(vma, - (dw * real_page_count(obj)) << PAGE_SHIFT | - (dw * sizeof(u32)), - real_page_count(obj), - dw); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto err_vma; - } - - rq = i915_request_alloc(engine, ctx); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_batch; - } - - flags = 0; - if (INTEL_GEN(vm->i915) <= 5) - flags |= I915_DISPATCH_SECURE; - - err = engine->emit_bb_start(rq, - batch->node.start, batch->node.size, - flags); - if (err) - goto err_request; - - err = i915_vma_move_to_active(batch, rq, 0); - if (err) - goto skip_request; - - err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); - if (err) - goto skip_request; - - i915_gem_object_set_active_reference(batch->obj); - i915_vma_unpin(batch); - i915_vma_close(batch); - - i915_vma_unpin(vma); - - i915_request_add(rq); - - return 0; - -skip_request: - i915_request_skip(rq, err); -err_request: - i915_request_add(rq); -err_batch: - i915_vma_unpin(batch); - i915_vma_put(batch); -err_vma: - i915_vma_unpin(vma); - return err; -} - -static int cpu_fill(struct drm_i915_gem_object *obj, u32 value) -{ - const bool has_llc = HAS_LLC(to_i915(obj->base.dev)); - unsigned int n, m, need_flush; - int err; - - err = i915_gem_obj_prepare_shmem_write(obj, &need_flush); - if (err) - return err; - - for (n = 0; n < real_page_count(obj); n++) { - u32 *map; - - map = kmap_atomic(i915_gem_object_get_page(obj, n)); - for (m = 0; m < DW_PER_PAGE; m++) - map[m] = value; - if (!has_llc) - drm_clflush_virt_range(map, PAGE_SIZE); - kunmap_atomic(map); - } - - i915_gem_obj_finish_shmem_access(obj); - obj->read_domains = I915_GEM_DOMAIN_GTT | I915_GEM_DOMAIN_CPU; - obj->write_domain = 0; - return 0; -} - -static noinline int cpu_check(struct drm_i915_gem_object *obj, - unsigned int idx, unsigned int max) -{ - unsigned int n, m, needs_flush; - int err; - - err = i915_gem_obj_prepare_shmem_read(obj, &needs_flush); - if (err) - return err; - - for (n = 0; n < real_page_count(obj); n++) { - u32 *map; - - map = kmap_atomic(i915_gem_object_get_page(obj, n)); - if (needs_flush & CLFLUSH_BEFORE) - drm_clflush_virt_range(map, PAGE_SIZE); - - for (m = 0; m < max; m++) { - if (map[m] != m) { - pr_err("%pS: Invalid value at object %d page %d/%ld, offset %d/%d: found %x expected %x\n", - __builtin_return_address(0), idx, - n, real_page_count(obj), m, max, - map[m], m); - err = -EINVAL; - goto out_unmap; - } - } - - for (; m < DW_PER_PAGE; m++) { - if (map[m] != STACK_MAGIC) { - pr_err("%pS: Invalid value at object %d page %d, offset %d: found %x expected %x (uninitialised)\n", - __builtin_return_address(0), idx, n, m, - map[m], STACK_MAGIC); - err = -EINVAL; - goto out_unmap; - } - } - -out_unmap: - kunmap_atomic(map); - if (err) - break; - } - - i915_gem_obj_finish_shmem_access(obj); - return err; -} - -static int file_add_object(struct drm_file *file, - struct drm_i915_gem_object *obj) -{ - int err; - - GEM_BUG_ON(obj->base.handle_count); - - /* tie the object to the drm_file for easy reaping */ - err = idr_alloc(&file->object_idr, &obj->base, 1, 0, GFP_KERNEL); - if (err < 0) - return err; - - i915_gem_object_get(obj); - obj->base.handle_count++; - return 0; -} - -static struct drm_i915_gem_object * -create_test_object(struct i915_gem_context *ctx, - struct drm_file *file, - struct list_head *objects) -{ - struct drm_i915_gem_object *obj; - struct i915_address_space *vm = - ctx->ppgtt ? &ctx->ppgtt->vm : &ctx->i915->ggtt.vm; - u64 size; - int err; - - size = min(vm->total / 2, 1024ull * DW_PER_PAGE * PAGE_SIZE); - size = round_down(size, DW_PER_PAGE * PAGE_SIZE); - - obj = huge_gem_object(ctx->i915, DW_PER_PAGE * PAGE_SIZE, size); - if (IS_ERR(obj)) - return obj; - - err = file_add_object(file, obj); - i915_gem_object_put(obj); - if (err) - return ERR_PTR(err); - - err = cpu_fill(obj, STACK_MAGIC); - if (err) { - pr_err("Failed to fill object with cpu, err=%d\n", - err); - return ERR_PTR(err); - } - - list_add_tail(&obj->st_link, objects); - return obj; -} - -static unsigned long max_dwords(struct drm_i915_gem_object *obj) -{ - unsigned long npages = fake_page_count(obj); - - GEM_BUG_ON(!IS_ALIGNED(npages, DW_PER_PAGE)); - return npages / DW_PER_PAGE; -} - -static int igt_ctx_exec(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *engine; - enum intel_engine_id id; - int err = -ENODEV; - - /* - * Create a few different contexts (with different mm) and write - * through each ctx/mm using the GPU making sure those writes end - * up in the expected pages of our obj. - */ - - if (!DRIVER_CAPS(i915)->has_logical_contexts) - return 0; - - for_each_engine(engine, i915, id) { - struct drm_i915_gem_object *obj = NULL; - unsigned long ncontexts, ndwords, dw; - struct igt_live_test t; - struct drm_file *file; - IGT_TIMEOUT(end_time); - LIST_HEAD(objects); - - if (!intel_engine_can_store_dword(engine)) - continue; - - if (!engine->context_size) - continue; /* No logical context support in HW */ - - file = mock_file(i915); - if (IS_ERR(file)) - return PTR_ERR(file); - - mutex_lock(&i915->drm.struct_mutex); - - err = igt_live_test_begin(&t, i915, __func__, engine->name); - if (err) - goto out_unlock; - - ncontexts = 0; - ndwords = 0; - dw = 0; - while (!time_after(jiffies, end_time)) { - struct i915_gem_context *ctx; - intel_wakeref_t wakeref; - - ctx = live_context(i915, file); - if (IS_ERR(ctx)) { - err = PTR_ERR(ctx); - goto out_unlock; - } - - if (!obj) { - obj = create_test_object(ctx, file, &objects); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto out_unlock; - } - } - - with_intel_runtime_pm(i915, wakeref) - err = gpu_fill(obj, ctx, engine, dw); - if (err) { - pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n", - ndwords, dw, max_dwords(obj), - engine->name, ctx->hw_id, - yesno(!!ctx->ppgtt), err); - goto out_unlock; - } - - if (++dw == max_dwords(obj)) { - obj = NULL; - dw = 0; - } - - ndwords++; - ncontexts++; - } - - pr_info("Submitted %lu contexts to %s, filling %lu dwords\n", - ncontexts, engine->name, ndwords); - - ncontexts = dw = 0; - list_for_each_entry(obj, &objects, st_link) { - unsigned int rem = - min_t(unsigned int, ndwords - dw, max_dwords(obj)); - - err = cpu_check(obj, ncontexts++, rem); - if (err) - break; - - dw += rem; - } - -out_unlock: - if (igt_live_test_end(&t)) - err = -EIO; - mutex_unlock(&i915->drm.struct_mutex); - - mock_file_free(i915, file); - if (err) - return err; - } - - return 0; -} - -static int igt_shared_ctx_exec(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct i915_gem_context *parent; - struct intel_engine_cs *engine; - enum intel_engine_id id; - struct igt_live_test t; - struct drm_file *file; - int err = 0; - - /* - * Create a few different contexts with the same mm and write - * through each ctx using the GPU making sure those writes end - * up in the expected pages of our obj. - */ - if (!DRIVER_CAPS(i915)->has_logical_contexts) - return 0; - - file = mock_file(i915); - if (IS_ERR(file)) - return PTR_ERR(file); - - mutex_lock(&i915->drm.struct_mutex); - - parent = live_context(i915, file); - if (IS_ERR(parent)) { - err = PTR_ERR(parent); - goto out_unlock; - } - - if (!parent->ppgtt) { /* not full-ppgtt; nothing to share */ - err = 0; - goto out_unlock; - } - - err = igt_live_test_begin(&t, i915, __func__, ""); - if (err) - goto out_unlock; - - for_each_engine(engine, i915, id) { - unsigned long ncontexts, ndwords, dw; - struct drm_i915_gem_object *obj = NULL; - IGT_TIMEOUT(end_time); - LIST_HEAD(objects); - - if (!intel_engine_can_store_dword(engine)) - continue; - - dw = 0; - ndwords = 0; - ncontexts = 0; - while (!time_after(jiffies, end_time)) { - struct i915_gem_context *ctx; - intel_wakeref_t wakeref; - - ctx = kernel_context(i915); - if (IS_ERR(ctx)) { - err = PTR_ERR(ctx); - goto out_test; - } - - __assign_ppgtt(ctx, parent->ppgtt); - - if (!obj) { - obj = create_test_object(parent, file, &objects); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - kernel_context_close(ctx); - goto out_test; - } - } - - err = 0; - with_intel_runtime_pm(i915, wakeref) - err = gpu_fill(obj, ctx, engine, dw); - if (err) { - pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n", - ndwords, dw, max_dwords(obj), - engine->name, ctx->hw_id, - yesno(!!ctx->ppgtt), err); - kernel_context_close(ctx); - goto out_test; - } - - if (++dw == max_dwords(obj)) { - obj = NULL; - dw = 0; - } - - ndwords++; - ncontexts++; - - kernel_context_close(ctx); - } - pr_info("Submitted %lu contexts to %s, filling %lu dwords\n", - ncontexts, engine->name, ndwords); - - ncontexts = dw = 0; - list_for_each_entry(obj, &objects, st_link) { - unsigned int rem = - min_t(unsigned int, ndwords - dw, max_dwords(obj)); - - err = cpu_check(obj, ncontexts++, rem); - if (err) - goto out_test; - - dw += rem; - } - } -out_test: - if (igt_live_test_end(&t)) - err = -EIO; -out_unlock: - mutex_unlock(&i915->drm.struct_mutex); - - mock_file_free(i915, file); - return err; -} - -static struct i915_vma *rpcs_query_batch(struct i915_vma *vma) -{ - struct drm_i915_gem_object *obj; - u32 *cmd; - int err; - - if (INTEL_GEN(vma->vm->i915) < 8) - return ERR_PTR(-EINVAL); - - obj = i915_gem_object_create_internal(vma->vm->i915, PAGE_SIZE); - if (IS_ERR(obj)) - return ERR_CAST(obj); - - cmd = i915_gem_object_pin_map(obj, I915_MAP_WB); - if (IS_ERR(cmd)) { - err = PTR_ERR(cmd); - goto err; - } - - *cmd++ = MI_STORE_REGISTER_MEM_GEN8; - *cmd++ = i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE); - *cmd++ = lower_32_bits(vma->node.start); - *cmd++ = upper_32_bits(vma->node.start); - *cmd = MI_BATCH_BUFFER_END; - - __i915_gem_object_flush_map(obj, 0, 64); - i915_gem_object_unpin_map(obj); - - vma = i915_vma_instance(obj, vma->vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto err; - } - - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) - goto err; - - return vma; - -err: - i915_gem_object_put(obj); - return ERR_PTR(err); -} - -static int -emit_rpcs_query(struct drm_i915_gem_object *obj, - struct i915_gem_context *ctx, - struct intel_engine_cs *engine, - struct i915_request **rq_out) -{ - struct i915_request *rq; - struct i915_vma *batch; - struct i915_vma *vma; - int err; - - GEM_BUG_ON(!intel_engine_can_store_dword(engine)); - - vma = i915_vma_instance(obj, &ctx->ppgtt->vm, NULL); - if (IS_ERR(vma)) - return PTR_ERR(vma); - - err = i915_gem_object_set_to_gtt_domain(obj, false); - if (err) - return err; - - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) - return err; - - batch = rpcs_query_batch(vma); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto err_vma; - } - - rq = i915_request_alloc(engine, ctx); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_batch; - } - - err = engine->emit_bb_start(rq, batch->node.start, batch->node.size, 0); - if (err) - goto err_request; - - err = i915_vma_move_to_active(batch, rq, 0); - if (err) - goto skip_request; - - err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); - if (err) - goto skip_request; - - i915_gem_object_set_active_reference(batch->obj); - i915_vma_unpin(batch); - i915_vma_close(batch); - - i915_vma_unpin(vma); - - *rq_out = i915_request_get(rq); - - i915_request_add(rq); - - return 0; - -skip_request: - i915_request_skip(rq, err); -err_request: - i915_request_add(rq); -err_batch: - i915_vma_unpin(batch); -err_vma: - i915_vma_unpin(vma); - - return err; -} - -#define TEST_IDLE BIT(0) -#define TEST_BUSY BIT(1) -#define TEST_RESET BIT(2) - -static int -__sseu_prepare(struct drm_i915_private *i915, - const char *name, - unsigned int flags, - struct i915_gem_context *ctx, - struct intel_engine_cs *engine, - struct igt_spinner **spin) -{ - struct i915_request *rq; - int ret; - - *spin = NULL; - if (!(flags & (TEST_BUSY | TEST_RESET))) - return 0; - - *spin = kzalloc(sizeof(**spin), GFP_KERNEL); - if (!*spin) - return -ENOMEM; - - ret = igt_spinner_init(*spin, i915); - if (ret) - goto err_free; - - rq = igt_spinner_create_request(*spin, ctx, engine, MI_NOOP); - if (IS_ERR(rq)) { - ret = PTR_ERR(rq); - goto err_fini; - } - - i915_request_add(rq); - - if (!igt_wait_for_spinner(*spin, rq)) { - pr_err("%s: Spinner failed to start!\n", name); - ret = -ETIMEDOUT; - goto err_end; - } - - return 0; - -err_end: - igt_spinner_end(*spin); -err_fini: - igt_spinner_fini(*spin); -err_free: - kfree(fetch_and_zero(spin)); - return ret; -} - -static int -__read_slice_count(struct drm_i915_private *i915, - struct i915_gem_context *ctx, - struct intel_engine_cs *engine, - struct drm_i915_gem_object *obj, - struct igt_spinner *spin, - u32 *rpcs) -{ - struct i915_request *rq = NULL; - u32 s_mask, s_shift; - unsigned int cnt; - u32 *buf, val; - long ret; - - ret = emit_rpcs_query(obj, ctx, engine, &rq); - if (ret) - return ret; - - if (spin) - igt_spinner_end(spin); - - ret = i915_request_wait(rq, I915_WAIT_LOCKED, MAX_SCHEDULE_TIMEOUT); - i915_request_put(rq); - if (ret < 0) - return ret; - - buf = i915_gem_object_pin_map(obj, I915_MAP_WB); - if (IS_ERR(buf)) { - ret = PTR_ERR(buf); - return ret; - } - - if (INTEL_GEN(i915) >= 11) { - s_mask = GEN11_RPCS_S_CNT_MASK; - s_shift = GEN11_RPCS_S_CNT_SHIFT; - } else { - s_mask = GEN8_RPCS_S_CNT_MASK; - s_shift = GEN8_RPCS_S_CNT_SHIFT; - } - - val = *buf; - cnt = (val & s_mask) >> s_shift; - *rpcs = val; - - i915_gem_object_unpin_map(obj); - - return cnt; -} - -static int -__check_rpcs(const char *name, u32 rpcs, int slices, unsigned int expected, - const char *prefix, const char *suffix) -{ - if (slices == expected) - return 0; - - if (slices < 0) { - pr_err("%s: %s read slice count failed with %d%s\n", - name, prefix, slices, suffix); - return slices; - } - - pr_err("%s: %s slice count %d is not %u%s\n", - name, prefix, slices, expected, suffix); - - pr_info("RPCS=0x%x; %u%sx%u%s\n", - rpcs, slices, - (rpcs & GEN8_RPCS_S_CNT_ENABLE) ? "*" : "", - (rpcs & GEN8_RPCS_SS_CNT_MASK) >> GEN8_RPCS_SS_CNT_SHIFT, - (rpcs & GEN8_RPCS_SS_CNT_ENABLE) ? "*" : ""); - - return -EINVAL; -} - -static int -__sseu_finish(struct drm_i915_private *i915, - const char *name, - unsigned int flags, - struct i915_gem_context *ctx, - struct i915_gem_context *kctx, - struct intel_engine_cs *engine, - struct drm_i915_gem_object *obj, - unsigned int expected, - struct igt_spinner *spin) -{ - unsigned int slices = - hweight32(intel_device_default_sseu(i915).slice_mask); - u32 rpcs = 0; - int ret = 0; - - if (flags & TEST_RESET) { - ret = i915_reset_engine(engine, "sseu"); - if (ret) - goto out; - } - - ret = __read_slice_count(i915, ctx, engine, obj, - flags & TEST_RESET ? NULL : spin, &rpcs); - ret = __check_rpcs(name, rpcs, ret, expected, "Context", "!"); - if (ret) - goto out; - - ret = __read_slice_count(i915, kctx, engine, obj, NULL, &rpcs); - ret = __check_rpcs(name, rpcs, ret, slices, "Kernel context", "!"); - -out: - if (spin) - igt_spinner_end(spin); - - if ((flags & TEST_IDLE) && ret == 0) { - ret = i915_gem_wait_for_idle(i915, - I915_WAIT_LOCKED, - MAX_SCHEDULE_TIMEOUT); - if (ret) - return ret; - - ret = __read_slice_count(i915, ctx, engine, obj, NULL, &rpcs); - ret = __check_rpcs(name, rpcs, ret, expected, - "Context", " after idle!"); - } - - return ret; -} - -static int -__sseu_test(struct drm_i915_private *i915, - const char *name, - unsigned int flags, - struct i915_gem_context *ctx, - struct intel_engine_cs *engine, - struct drm_i915_gem_object *obj, - struct intel_sseu sseu) -{ - struct igt_spinner *spin = NULL; - struct i915_gem_context *kctx; - int ret; - - kctx = kernel_context(i915); - if (IS_ERR(kctx)) - return PTR_ERR(kctx); - - ret = __sseu_prepare(i915, name, flags, ctx, engine, &spin); - if (ret) - goto out_context; - - ret = __i915_gem_context_reconfigure_sseu(ctx, engine, sseu); - if (ret) - goto out_spin; - - ret = __sseu_finish(i915, name, flags, ctx, kctx, engine, obj, - hweight32(sseu.slice_mask), spin); - -out_spin: - if (spin) { - igt_spinner_end(spin); - igt_spinner_fini(spin); - kfree(spin); - } - -out_context: - kernel_context_close(kctx); - - return ret; -} - -static int -__igt_ctx_sseu(struct drm_i915_private *i915, - const char *name, - unsigned int flags) -{ - struct intel_sseu default_sseu = intel_device_default_sseu(i915); - struct intel_engine_cs *engine = i915->engine[RCS0]; - struct drm_i915_gem_object *obj; - struct i915_gem_context *ctx; - struct intel_sseu pg_sseu; - intel_wakeref_t wakeref; - struct drm_file *file; - int ret; - - if (INTEL_GEN(i915) < 9) - return 0; - - if (!RUNTIME_INFO(i915)->sseu.has_slice_pg) - return 0; - - if (hweight32(default_sseu.slice_mask) < 2) - return 0; - - /* - * Gen11 VME friendly power-gated configuration with half enabled - * sub-slices. - */ - pg_sseu = default_sseu; - pg_sseu.slice_mask = 1; - pg_sseu.subslice_mask = - ~(~0 << (hweight32(default_sseu.subslice_mask) / 2)); - - pr_info("SSEU subtest '%s', flags=%x, def_slices=%u, pg_slices=%u\n", - name, flags, hweight32(default_sseu.slice_mask), - hweight32(pg_sseu.slice_mask)); - - file = mock_file(i915); - if (IS_ERR(file)) - return PTR_ERR(file); - - if (flags & TEST_RESET) - igt_global_reset_lock(i915); - - mutex_lock(&i915->drm.struct_mutex); - - ctx = live_context(i915, file); - if (IS_ERR(ctx)) { - ret = PTR_ERR(ctx); - goto out_unlock; - } - i915_gem_context_clear_bannable(ctx); /* to reset and beyond! */ - - obj = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(obj)) { - ret = PTR_ERR(obj); - goto out_unlock; - } - - wakeref = intel_runtime_pm_get(i915); - - /* First set the default mask. */ - ret = __sseu_test(i915, name, flags, ctx, engine, obj, default_sseu); - if (ret) - goto out_fail; - - /* Then set a power-gated configuration. */ - ret = __sseu_test(i915, name, flags, ctx, engine, obj, pg_sseu); - if (ret) - goto out_fail; - - /* Back to defaults. */ - ret = __sseu_test(i915, name, flags, ctx, engine, obj, default_sseu); - if (ret) - goto out_fail; - - /* One last power-gated configuration for the road. */ - ret = __sseu_test(i915, name, flags, ctx, engine, obj, pg_sseu); - if (ret) - goto out_fail; - -out_fail: - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - ret = -EIO; - - i915_gem_object_put(obj); - - intel_runtime_pm_put(i915, wakeref); - -out_unlock: - mutex_unlock(&i915->drm.struct_mutex); - - if (flags & TEST_RESET) - igt_global_reset_unlock(i915); - - mock_file_free(i915, file); - - if (ret) - pr_err("%s: Failed with %d!\n", name, ret); - - return ret; -} - -static int igt_ctx_sseu(void *arg) -{ - struct { - const char *name; - unsigned int flags; - } *phase, phases[] = { - { .name = "basic", .flags = 0 }, - { .name = "idle", .flags = TEST_IDLE }, - { .name = "busy", .flags = TEST_BUSY }, - { .name = "busy-reset", .flags = TEST_BUSY | TEST_RESET }, - { .name = "busy-idle", .flags = TEST_BUSY | TEST_IDLE }, - { .name = "reset-idle", .flags = TEST_RESET | TEST_IDLE }, - }; - unsigned int i; - int ret = 0; - - for (i = 0, phase = phases; ret == 0 && i < ARRAY_SIZE(phases); - i++, phase++) - ret = __igt_ctx_sseu(arg, phase->name, phase->flags); - - return ret; -} - -static int igt_ctx_readonly(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct drm_i915_gem_object *obj = NULL; - struct i915_gem_context *ctx; - struct i915_hw_ppgtt *ppgtt; - unsigned long idx, ndwords, dw; - struct igt_live_test t; - struct drm_file *file; - I915_RND_STATE(prng); - IGT_TIMEOUT(end_time); - LIST_HEAD(objects); - int err = -ENODEV; - - /* - * Create a few read-only objects (with the occasional writable object) - * and try to write into these object checking that the GPU discards - * any write to a read-only object. - */ - - file = mock_file(i915); - if (IS_ERR(file)) - return PTR_ERR(file); - - mutex_lock(&i915->drm.struct_mutex); - - err = igt_live_test_begin(&t, i915, __func__, ""); - if (err) - goto out_unlock; - - ctx = live_context(i915, file); - if (IS_ERR(ctx)) { - err = PTR_ERR(ctx); - goto out_unlock; - } - - ppgtt = ctx->ppgtt ?: i915->mm.aliasing_ppgtt; - if (!ppgtt || !ppgtt->vm.has_read_only) { - err = 0; - goto out_unlock; - } - - ndwords = 0; - dw = 0; - while (!time_after(jiffies, end_time)) { - struct intel_engine_cs *engine; - unsigned int id; - - for_each_engine(engine, i915, id) { - intel_wakeref_t wakeref; - - if (!intel_engine_can_store_dword(engine)) - continue; - - if (!obj) { - obj = create_test_object(ctx, file, &objects); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto out_unlock; - } - - if (prandom_u32_state(&prng) & 1) - i915_gem_object_set_readonly(obj); - } - - err = 0; - with_intel_runtime_pm(i915, wakeref) - err = gpu_fill(obj, ctx, engine, dw); - if (err) { - pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n", - ndwords, dw, max_dwords(obj), - engine->name, ctx->hw_id, - yesno(!!ctx->ppgtt), err); - goto out_unlock; - } - - if (++dw == max_dwords(obj)) { - obj = NULL; - dw = 0; - } - ndwords++; - } - } - pr_info("Submitted %lu dwords (across %u engines)\n", - ndwords, RUNTIME_INFO(i915)->num_engines); - - dw = 0; - idx = 0; - list_for_each_entry(obj, &objects, st_link) { - unsigned int rem = - min_t(unsigned int, ndwords - dw, max_dwords(obj)); - unsigned int num_writes; - - num_writes = rem; - if (i915_gem_object_is_readonly(obj)) - num_writes = 0; - - err = cpu_check(obj, idx++, num_writes); - if (err) - break; - - dw += rem; - } - -out_unlock: - if (igt_live_test_end(&t)) - err = -EIO; - mutex_unlock(&i915->drm.struct_mutex); - - mock_file_free(i915, file); - return err; -} - -static int check_scratch(struct i915_gem_context *ctx, u64 offset) -{ - struct drm_mm_node *node = - __drm_mm_interval_first(&ctx->ppgtt->vm.mm, - offset, offset + sizeof(u32) - 1); - if (!node || node->start > offset) - return 0; - - GEM_BUG_ON(offset >= node->start + node->size); - - pr_err("Target offset 0x%08x_%08x overlaps with a node in the mm!\n", - upper_32_bits(offset), lower_32_bits(offset)); - return -EINVAL; -} - -static int write_to_scratch(struct i915_gem_context *ctx, - struct intel_engine_cs *engine, - u64 offset, u32 value) -{ - struct drm_i915_private *i915 = ctx->i915; - struct drm_i915_gem_object *obj; - struct i915_request *rq; - struct i915_vma *vma; - u32 *cmd; - int err; - - GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE); - - obj = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - cmd = i915_gem_object_pin_map(obj, I915_MAP_WB); - if (IS_ERR(cmd)) { - err = PTR_ERR(cmd); - goto err; - } - - *cmd++ = MI_STORE_DWORD_IMM_GEN4; - if (INTEL_GEN(i915) >= 8) { - *cmd++ = lower_32_bits(offset); - *cmd++ = upper_32_bits(offset); - } else { - *cmd++ = 0; - *cmd++ = offset; - } - *cmd++ = value; - *cmd = MI_BATCH_BUFFER_END; - __i915_gem_object_flush_map(obj, 0, 64); - i915_gem_object_unpin_map(obj); - - vma = i915_vma_instance(obj, &ctx->ppgtt->vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto err; - } - - err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED); - if (err) - goto err; - - err = check_scratch(ctx, offset); - if (err) - goto err_unpin; - - rq = i915_request_alloc(engine, ctx); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_unpin; - } - - err = engine->emit_bb_start(rq, vma->node.start, vma->node.size, 0); - if (err) - goto err_request; - - err = i915_vma_move_to_active(vma, rq, 0); - if (err) - goto skip_request; - - i915_gem_object_set_active_reference(obj); - i915_vma_unpin(vma); - i915_vma_close(vma); - - i915_request_add(rq); - - return 0; - -skip_request: - i915_request_skip(rq, err); -err_request: - i915_request_add(rq); -err_unpin: - i915_vma_unpin(vma); -err: - i915_gem_object_put(obj); - return err; -} - -static int read_from_scratch(struct i915_gem_context *ctx, - struct intel_engine_cs *engine, - u64 offset, u32 *value) -{ - struct drm_i915_private *i915 = ctx->i915; - struct drm_i915_gem_object *obj; - const u32 RCS_GPR0 = 0x2600; /* not all engines have their own GPR! */ - const u32 result = 0x100; - struct i915_request *rq; - struct i915_vma *vma; - u32 *cmd; - int err; - - GEM_BUG_ON(offset < I915_GTT_PAGE_SIZE); - - obj = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - cmd = i915_gem_object_pin_map(obj, I915_MAP_WB); - if (IS_ERR(cmd)) { - err = PTR_ERR(cmd); - goto err; - } - - memset(cmd, POISON_INUSE, PAGE_SIZE); - if (INTEL_GEN(i915) >= 8) { - *cmd++ = MI_LOAD_REGISTER_MEM_GEN8; - *cmd++ = RCS_GPR0; - *cmd++ = lower_32_bits(offset); - *cmd++ = upper_32_bits(offset); - *cmd++ = MI_STORE_REGISTER_MEM_GEN8; - *cmd++ = RCS_GPR0; - *cmd++ = result; - *cmd++ = 0; - } else { - *cmd++ = MI_LOAD_REGISTER_MEM; - *cmd++ = RCS_GPR0; - *cmd++ = offset; - *cmd++ = MI_STORE_REGISTER_MEM; - *cmd++ = RCS_GPR0; - *cmd++ = result; - } - *cmd = MI_BATCH_BUFFER_END; - - i915_gem_object_flush_map(obj); - i915_gem_object_unpin_map(obj); - - vma = i915_vma_instance(obj, &ctx->ppgtt->vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto err; - } - - err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED); - if (err) - goto err; - - err = check_scratch(ctx, offset); - if (err) - goto err_unpin; - - rq = i915_request_alloc(engine, ctx); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_unpin; - } - - err = engine->emit_bb_start(rq, vma->node.start, vma->node.size, 0); - if (err) - goto err_request; - - err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); - if (err) - goto skip_request; - - i915_vma_unpin(vma); - i915_vma_close(vma); - - i915_request_add(rq); - - err = i915_gem_object_set_to_cpu_domain(obj, false); - if (err) - goto err; - - cmd = i915_gem_object_pin_map(obj, I915_MAP_WB); - if (IS_ERR(cmd)) { - err = PTR_ERR(cmd); - goto err; - } - - *value = cmd[result / sizeof(*cmd)]; - i915_gem_object_unpin_map(obj); - i915_gem_object_put(obj); - - return 0; - -skip_request: - i915_request_skip(rq, err); -err_request: - i915_request_add(rq); -err_unpin: - i915_vma_unpin(vma); -err: - i915_gem_object_put(obj); - return err; -} - -static int igt_vm_isolation(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct i915_gem_context *ctx_a, *ctx_b; - struct intel_engine_cs *engine; - intel_wakeref_t wakeref; - struct igt_live_test t; - struct drm_file *file; - I915_RND_STATE(prng); - unsigned long count; - unsigned int id; - u64 vm_total; - int err; - - if (INTEL_GEN(i915) < 7) - return 0; - - /* - * The simple goal here is that a write into one context is not - * observed in a second (separate page tables and scratch). - */ - - file = mock_file(i915); - if (IS_ERR(file)) - return PTR_ERR(file); - - mutex_lock(&i915->drm.struct_mutex); - - err = igt_live_test_begin(&t, i915, __func__, ""); - if (err) - goto out_unlock; - - ctx_a = live_context(i915, file); - if (IS_ERR(ctx_a)) { - err = PTR_ERR(ctx_a); - goto out_unlock; - } - - ctx_b = live_context(i915, file); - if (IS_ERR(ctx_b)) { - err = PTR_ERR(ctx_b); - goto out_unlock; - } - - /* We can only test vm isolation, if the vm are distinct */ - if (ctx_a->ppgtt == ctx_b->ppgtt) - goto out_unlock; - - vm_total = ctx_a->ppgtt->vm.total; - GEM_BUG_ON(ctx_b->ppgtt->vm.total != vm_total); - vm_total -= I915_GTT_PAGE_SIZE; - - wakeref = intel_runtime_pm_get(i915); - - count = 0; - for_each_engine(engine, i915, id) { - IGT_TIMEOUT(end_time); - unsigned long this = 0; - - if (!intel_engine_can_store_dword(engine)) - continue; - - while (!__igt_timeout(end_time, NULL)) { - u32 value = 0xc5c5c5c5; - u64 offset; - - div64_u64_rem(i915_prandom_u64_state(&prng), - vm_total, &offset); - offset &= -sizeof(u32); - offset += I915_GTT_PAGE_SIZE; - - err = write_to_scratch(ctx_a, engine, - offset, 0xdeadbeef); - if (err == 0) - err = read_from_scratch(ctx_b, engine, - offset, &value); - if (err) - goto out_rpm; - - if (value) { - pr_err("%s: Read %08x from scratch (offset 0x%08x_%08x), after %lu reads!\n", - engine->name, value, - upper_32_bits(offset), - lower_32_bits(offset), - this); - err = -EINVAL; - goto out_rpm; - } - - this++; - } - count += this; - } - pr_info("Checked %lu scratch offsets across %d engines\n", - count, RUNTIME_INFO(i915)->num_engines); - -out_rpm: - intel_runtime_pm_put(i915, wakeref); -out_unlock: - if (igt_live_test_end(&t)) - err = -EIO; - mutex_unlock(&i915->drm.struct_mutex); - - mock_file_free(i915, file); - return err; -} - -static __maybe_unused const char * -__engine_name(struct drm_i915_private *i915, intel_engine_mask_t engines) -{ - struct intel_engine_cs *engine; - intel_engine_mask_t tmp; - - if (engines == ALL_ENGINES) - return "all"; - - for_each_engine_masked(engine, i915, engines, tmp) - return engine->name; - - return "none"; -} - -static int __igt_switch_to_kernel_context(struct drm_i915_private *i915, - struct i915_gem_context *ctx, - intel_engine_mask_t engines) -{ - struct intel_engine_cs *engine; - intel_engine_mask_t tmp; - int pass; - - GEM_TRACE("Testing %s\n", __engine_name(i915, engines)); - for (pass = 0; pass < 4; pass++) { /* Once busy; once idle; repeat */ - bool from_idle = pass & 1; - int err; - - if (!from_idle) { - for_each_engine_masked(engine, i915, engines, tmp) { - struct i915_request *rq; - - rq = i915_request_alloc(engine, ctx); - if (IS_ERR(rq)) - return PTR_ERR(rq); - - i915_request_add(rq); - } - } - - err = i915_gem_switch_to_kernel_context(i915, - i915->gt.active_engines); - if (err) - return err; - - if (!from_idle) { - err = i915_gem_wait_for_idle(i915, - I915_WAIT_LOCKED, - MAX_SCHEDULE_TIMEOUT); - if (err) - return err; - } - - if (i915->gt.active_requests) { - pr_err("%d active requests remain after switching to kernel context, pass %d (%s) on %s engine%s\n", - i915->gt.active_requests, - pass, from_idle ? "idle" : "busy", - __engine_name(i915, engines), - is_power_of_2(engines) ? "" : "s"); - return -EINVAL; - } - - /* XXX Bonus points for proving we are the kernel context! */ - - mutex_unlock(&i915->drm.struct_mutex); - drain_delayed_work(&i915->gt.idle_work); - mutex_lock(&i915->drm.struct_mutex); - } - - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - return -EIO; - - return 0; -} - -static int igt_switch_to_kernel_context(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *engine; - struct i915_gem_context *ctx; - enum intel_engine_id id; - intel_wakeref_t wakeref; - int err; - - /* - * A core premise of switching to the kernel context is that - * if an engine is already idling in the kernel context, we - * do not emit another request and wake it up. The other being - * that we do indeed end up idling in the kernel context. - */ - - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); - - ctx = kernel_context(i915); - if (IS_ERR(ctx)) { - mutex_unlock(&i915->drm.struct_mutex); - return PTR_ERR(ctx); - } - - /* First check idling each individual engine */ - for_each_engine(engine, i915, id) { - err = __igt_switch_to_kernel_context(i915, ctx, BIT(id)); - if (err) - goto out_unlock; - } - - /* Now en masse */ - err = __igt_switch_to_kernel_context(i915, ctx, ALL_ENGINES); - if (err) - goto out_unlock; - -out_unlock: - GEM_TRACE_DUMP_ON(err); - - intel_runtime_pm_put(i915, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - - kernel_context_close(ctx); - return err; -} - -static void mock_barrier_task(void *data) -{ - unsigned int *counter = data; - - ++*counter; -} - -static int mock_context_barrier(void *arg) -{ -#undef pr_fmt -#define pr_fmt(x) "context_barrier_task():" # x - struct drm_i915_private *i915 = arg; - struct i915_gem_context *ctx; - struct i915_request *rq; - intel_wakeref_t wakeref; - unsigned int counter; - int err; - - /* - * The context barrier provides us with a callback after it emits - * a request; useful for retiring old state after loading new. - */ - - mutex_lock(&i915->drm.struct_mutex); - - ctx = mock_context(i915, "mock"); - if (!ctx) { - err = -ENOMEM; - goto unlock; - } - - counter = 0; - err = context_barrier_task(ctx, 0, - NULL, mock_barrier_task, &counter); - if (err) { - pr_err("Failed at line %d, err=%d\n", __LINE__, err); - goto out; - } - if (counter == 0) { - pr_err("Did not retire immediately with 0 engines\n"); - err = -EINVAL; - goto out; - } - - counter = 0; - err = context_barrier_task(ctx, ALL_ENGINES, - NULL, mock_barrier_task, &counter); - if (err) { - pr_err("Failed at line %d, err=%d\n", __LINE__, err); - goto out; - } - if (counter == 0) { - pr_err("Did not retire immediately for all inactive engines\n"); - err = -EINVAL; - goto out; - } - - rq = ERR_PTR(-ENODEV); - with_intel_runtime_pm(i915, wakeref) - rq = i915_request_alloc(i915->engine[RCS0], ctx); - if (IS_ERR(rq)) { - pr_err("Request allocation failed!\n"); - goto out; - } - i915_request_add(rq); - GEM_BUG_ON(list_empty(&ctx->active_engines)); - - counter = 0; - context_barrier_inject_fault = BIT(RCS0); - err = context_barrier_task(ctx, ALL_ENGINES, - NULL, mock_barrier_task, &counter); - context_barrier_inject_fault = 0; - if (err == -ENXIO) - err = 0; - else - pr_err("Did not hit fault injection!\n"); - if (counter != 0) { - pr_err("Invoked callback on error!\n"); - err = -EIO; - } - if (err) - goto out; - - counter = 0; - err = context_barrier_task(ctx, ALL_ENGINES, - NULL, mock_barrier_task, &counter); - if (err) { - pr_err("Failed at line %d, err=%d\n", __LINE__, err); - goto out; - } - mock_device_flush(i915); - if (counter == 0) { - pr_err("Did not retire on each active engines\n"); - err = -EINVAL; - goto out; - } - -out: - mock_context_close(ctx); -unlock: - mutex_unlock(&i915->drm.struct_mutex); - return err; -#undef pr_fmt -#define pr_fmt(x) x -} - -int i915_gem_context_mock_selftests(void) -{ - static const struct i915_subtest tests[] = { - SUBTEST(igt_switch_to_kernel_context), - SUBTEST(mock_context_barrier), - }; - struct drm_i915_private *i915; - int err; - - i915 = mock_gem_device(); - if (!i915) - return -ENOMEM; - - err = i915_subtests(tests, i915); - - drm_dev_put(&i915->drm); - return err; -} - -int i915_gem_context_live_selftests(struct drm_i915_private *dev_priv) -{ - static const struct i915_subtest tests[] = { - SUBTEST(igt_switch_to_kernel_context), - SUBTEST(live_nop_switch), - SUBTEST(igt_ctx_exec), - SUBTEST(igt_ctx_readonly), - SUBTEST(igt_ctx_sseu), - SUBTEST(igt_shared_ctx_exec), - SUBTEST(igt_vm_isolation), - }; - - if (i915_terminally_wedged(dev_priv)) - return 0; - - return i915_subtests(tests, dev_priv); -} diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c deleted file mode 100644 index 2b943ee246c9..000000000000 --- a/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c +++ /dev/null @@ -1,404 +0,0 @@ -/* - * Copyright © 2016 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#include "../i915_selftest.h" - -#include "mock_gem_device.h" -#include "mock_dmabuf.h" - -static int igt_dmabuf_export(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct drm_i915_gem_object *obj; - struct dma_buf *dmabuf; - - obj = i915_gem_object_create(i915, PAGE_SIZE); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - dmabuf = i915_gem_prime_export(&i915->drm, &obj->base, 0); - i915_gem_object_put(obj); - if (IS_ERR(dmabuf)) { - pr_err("i915_gem_prime_export failed with err=%d\n", - (int)PTR_ERR(dmabuf)); - return PTR_ERR(dmabuf); - } - - dma_buf_put(dmabuf); - return 0; -} - -static int igt_dmabuf_import_self(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct drm_i915_gem_object *obj; - struct drm_gem_object *import; - struct dma_buf *dmabuf; - int err; - - obj = i915_gem_object_create(i915, PAGE_SIZE); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - dmabuf = i915_gem_prime_export(&i915->drm, &obj->base, 0); - if (IS_ERR(dmabuf)) { - pr_err("i915_gem_prime_export failed with err=%d\n", - (int)PTR_ERR(dmabuf)); - err = PTR_ERR(dmabuf); - goto out; - } - - import = i915_gem_prime_import(&i915->drm, dmabuf); - if (IS_ERR(import)) { - pr_err("i915_gem_prime_import failed with err=%d\n", - (int)PTR_ERR(import)); - err = PTR_ERR(import); - goto out_dmabuf; - } - - if (import != &obj->base) { - pr_err("i915_gem_prime_import created a new object!\n"); - err = -EINVAL; - goto out_import; - } - - err = 0; -out_import: - i915_gem_object_put(to_intel_bo(import)); -out_dmabuf: - dma_buf_put(dmabuf); -out: - i915_gem_object_put(obj); - return err; -} - -static int igt_dmabuf_import(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct drm_i915_gem_object *obj; - struct dma_buf *dmabuf; - void *obj_map, *dma_map; - u32 pattern[] = { 0, 0xaa, 0xcc, 0x55, 0xff }; - int err, i; - - dmabuf = mock_dmabuf(1); - if (IS_ERR(dmabuf)) - return PTR_ERR(dmabuf); - - obj = to_intel_bo(i915_gem_prime_import(&i915->drm, dmabuf)); - if (IS_ERR(obj)) { - pr_err("i915_gem_prime_import failed with err=%d\n", - (int)PTR_ERR(obj)); - err = PTR_ERR(obj); - goto out_dmabuf; - } - - if (obj->base.dev != &i915->drm) { - pr_err("i915_gem_prime_import created a non-i915 object!\n"); - err = -EINVAL; - goto out_obj; - } - - if (obj->base.size != PAGE_SIZE) { - pr_err("i915_gem_prime_import is wrong size found %lld, expected %ld\n", - (long long)obj->base.size, PAGE_SIZE); - err = -EINVAL; - goto out_obj; - } - - dma_map = dma_buf_vmap(dmabuf); - if (!dma_map) { - pr_err("dma_buf_vmap failed\n"); - err = -ENOMEM; - goto out_obj; - } - - if (0) { /* Can not yet map dmabuf */ - obj_map = i915_gem_object_pin_map(obj, I915_MAP_WB); - if (IS_ERR(obj_map)) { - err = PTR_ERR(obj_map); - pr_err("i915_gem_object_pin_map failed with err=%d\n", err); - goto out_dma_map; - } - - for (i = 0; i < ARRAY_SIZE(pattern); i++) { - memset(dma_map, pattern[i], PAGE_SIZE); - if (memchr_inv(obj_map, pattern[i], PAGE_SIZE)) { - err = -EINVAL; - pr_err("imported vmap not all set to %x!\n", pattern[i]); - i915_gem_object_unpin_map(obj); - goto out_dma_map; - } - } - - for (i = 0; i < ARRAY_SIZE(pattern); i++) { - memset(obj_map, pattern[i], PAGE_SIZE); - if (memchr_inv(dma_map, pattern[i], PAGE_SIZE)) { - err = -EINVAL; - pr_err("exported vmap not all set to %x!\n", pattern[i]); - i915_gem_object_unpin_map(obj); - goto out_dma_map; - } - } - - i915_gem_object_unpin_map(obj); - } - - err = 0; -out_dma_map: - dma_buf_vunmap(dmabuf, dma_map); -out_obj: - i915_gem_object_put(obj); -out_dmabuf: - dma_buf_put(dmabuf); - return err; -} - -static int igt_dmabuf_import_ownership(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct drm_i915_gem_object *obj; - struct dma_buf *dmabuf; - void *ptr; - int err; - - dmabuf = mock_dmabuf(1); - if (IS_ERR(dmabuf)) - return PTR_ERR(dmabuf); - - ptr = dma_buf_vmap(dmabuf); - if (!ptr) { - pr_err("dma_buf_vmap failed\n"); - err = -ENOMEM; - goto err_dmabuf; - } - - memset(ptr, 0xc5, PAGE_SIZE); - dma_buf_vunmap(dmabuf, ptr); - - obj = to_intel_bo(i915_gem_prime_import(&i915->drm, dmabuf)); - if (IS_ERR(obj)) { - pr_err("i915_gem_prime_import failed with err=%d\n", - (int)PTR_ERR(obj)); - err = PTR_ERR(obj); - goto err_dmabuf; - } - - dma_buf_put(dmabuf); - - err = i915_gem_object_pin_pages(obj); - if (err) { - pr_err("i915_gem_object_pin_pages failed with err=%d\n", err); - goto out_obj; - } - - err = 0; - i915_gem_object_unpin_pages(obj); -out_obj: - i915_gem_object_put(obj); - return err; - -err_dmabuf: - dma_buf_put(dmabuf); - return err; -} - -static int igt_dmabuf_export_vmap(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct drm_i915_gem_object *obj; - struct dma_buf *dmabuf; - void *ptr; - int err; - - obj = i915_gem_object_create(i915, PAGE_SIZE); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - dmabuf = i915_gem_prime_export(&i915->drm, &obj->base, 0); - if (IS_ERR(dmabuf)) { - pr_err("i915_gem_prime_export failed with err=%d\n", - (int)PTR_ERR(dmabuf)); - err = PTR_ERR(dmabuf); - goto err_obj; - } - i915_gem_object_put(obj); - - ptr = dma_buf_vmap(dmabuf); - if (!ptr) { - pr_err("dma_buf_vmap failed\n"); - err = -ENOMEM; - goto out; - } - - if (memchr_inv(ptr, 0, dmabuf->size)) { - pr_err("Exported object not initialiased to zero!\n"); - err = -EINVAL; - goto out; - } - - memset(ptr, 0xc5, dmabuf->size); - - err = 0; - dma_buf_vunmap(dmabuf, ptr); -out: - dma_buf_put(dmabuf); - return err; - -err_obj: - i915_gem_object_put(obj); - return err; -} - -static int igt_dmabuf_export_kmap(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct drm_i915_gem_object *obj; - struct dma_buf *dmabuf; - void *ptr; - int err; - - obj = i915_gem_object_create(i915, 2*PAGE_SIZE); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - dmabuf = i915_gem_prime_export(&i915->drm, &obj->base, 0); - i915_gem_object_put(obj); - if (IS_ERR(dmabuf)) { - err = PTR_ERR(dmabuf); - pr_err("i915_gem_prime_export failed with err=%d\n", err); - return err; - } - - ptr = dma_buf_kmap(dmabuf, 0); - if (!ptr) { - pr_err("dma_buf_kmap failed\n"); - err = -ENOMEM; - goto err; - } - - if (memchr_inv(ptr, 0, PAGE_SIZE)) { - dma_buf_kunmap(dmabuf, 0, ptr); - pr_err("Exported page[0] not initialiased to zero!\n"); - err = -EINVAL; - goto err; - } - - memset(ptr, 0xc5, PAGE_SIZE); - dma_buf_kunmap(dmabuf, 0, ptr); - - ptr = i915_gem_object_pin_map(obj, I915_MAP_WB); - if (IS_ERR(ptr)) { - err = PTR_ERR(ptr); - pr_err("i915_gem_object_pin_map failed with err=%d\n", err); - goto err; - } - memset(ptr + PAGE_SIZE, 0xaa, PAGE_SIZE); - i915_gem_object_flush_map(obj); - i915_gem_object_unpin_map(obj); - - ptr = dma_buf_kmap(dmabuf, 1); - if (!ptr) { - pr_err("dma_buf_kmap failed\n"); - err = -ENOMEM; - goto err; - } - - if (memchr_inv(ptr, 0xaa, PAGE_SIZE)) { - dma_buf_kunmap(dmabuf, 1, ptr); - pr_err("Exported page[1] not set to 0xaa!\n"); - err = -EINVAL; - goto err; - } - - memset(ptr, 0xc5, PAGE_SIZE); - dma_buf_kunmap(dmabuf, 1, ptr); - - ptr = dma_buf_kmap(dmabuf, 0); - if (!ptr) { - pr_err("dma_buf_kmap failed\n"); - err = -ENOMEM; - goto err; - } - if (memchr_inv(ptr, 0xc5, PAGE_SIZE)) { - dma_buf_kunmap(dmabuf, 0, ptr); - pr_err("Exported page[0] did not retain 0xc5!\n"); - err = -EINVAL; - goto err; - } - dma_buf_kunmap(dmabuf, 0, ptr); - - ptr = dma_buf_kmap(dmabuf, 2); - if (ptr) { - pr_err("Erroneously kmapped beyond the end of the object!\n"); - dma_buf_kunmap(dmabuf, 2, ptr); - err = -EINVAL; - goto err; - } - - ptr = dma_buf_kmap(dmabuf, -1); - if (ptr) { - pr_err("Erroneously kmapped before the start of the object!\n"); - dma_buf_kunmap(dmabuf, -1, ptr); - err = -EINVAL; - goto err; - } - - err = 0; -err: - dma_buf_put(dmabuf); - return err; -} - -int i915_gem_dmabuf_mock_selftests(void) -{ - static const struct i915_subtest tests[] = { - SUBTEST(igt_dmabuf_export), - SUBTEST(igt_dmabuf_import_self), - SUBTEST(igt_dmabuf_import), - SUBTEST(igt_dmabuf_import_ownership), - SUBTEST(igt_dmabuf_export_vmap), - SUBTEST(igt_dmabuf_export_kmap), - }; - struct drm_i915_private *i915; - int err; - - i915 = mock_gem_device(); - if (!i915) - return -ENOMEM; - - err = i915_subtests(tests, i915); - - drm_dev_put(&i915->drm); - return err; -} - -int i915_gem_dmabuf_live_selftests(struct drm_i915_private *i915) -{ - static const struct i915_subtest tests[] = { - SUBTEST(igt_dmabuf_export), - }; - - return i915_subtests(tests, i915); -} diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c index 89766688e420..a3cb0aade6f1 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c @@ -22,10 +22,14 @@ * */ -#include "../i915_selftest.h" +#include "gem/i915_gem_pm.h" +#include "gem/selftests/igt_gem_utils.h" +#include "gem/selftests/mock_context.h" +#include "i915_selftest.h" + +#include "igt_flush_test.h" #include "lib_sw_fence.h" -#include "mock_context.h" #include "mock_drm.h" #include "mock_gem_device.h" @@ -64,20 +68,24 @@ static int populate_ggtt(struct drm_i915_private *i915, count++; } + bound = 0; unbound = 0; - list_for_each_entry(obj, &i915->mm.unbound_list, mm.link) - if (obj->mm.quirked) + list_for_each_entry(obj, objects, st_link) { + GEM_BUG_ON(!obj->mm.quirked); + + if (atomic_read(&obj->bind_count)) + bound++; + else unbound++; + } + GEM_BUG_ON(bound + unbound != count); + if (unbound) { pr_err("%s: Found %lu objects unbound, expected %u!\n", __func__, unbound, 0); return -EINVAL; } - bound = 0; - list_for_each_entry(obj, &i915->mm.bound_list, mm.link) - if (obj->mm.quirked) - bound++; if (bound != count) { pr_err("%s: Found %lu objects bound, expected %lu!\n", __func__, bound, count); @@ -397,7 +405,7 @@ static int igt_evict_contexts(void *arg) return 0; mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); /* Reserve a block so that we know we have enough to fit a few rq */ memset(&hole, 0, sizeof(hole)); @@ -460,7 +468,7 @@ static int igt_evict_contexts(void *arg) /* We will need some GGTT space for the rq's context */ igt_evict_ctl.fail_if_busy = true; - rq = i915_request_alloc(engine, ctx); + rq = igt_request_alloc(ctx, engine); igt_evict_ctl.fail_if_busy = false; if (IS_ERR(rq)) { @@ -498,6 +506,8 @@ static int igt_evict_contexts(void *arg) mutex_lock(&i915->drm.struct_mutex); out_locked: + if (igt_flush_test(i915, I915_WAIT_LOCKED)) + err = -EIO; while (reserved) { struct reserved *next = reserved->next; @@ -508,7 +518,7 @@ out_locked: } if (drm_mm_node_allocated(&hole)) drm_mm_remove_node(&hole); - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; @@ -532,7 +542,7 @@ int i915_gem_evict_mock_selftests(void) return -ENOMEM; mutex_lock(&i915->drm.struct_mutex); - with_intel_runtime_pm(i915, wakeref) + with_intel_runtime_pm(&i915->runtime_pm, wakeref) err = i915_subtests(tests, i915); mutex_unlock(&i915->drm.struct_mutex); diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c index 9cca66e4420a..1a60b9fe8221 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c @@ -25,10 +25,11 @@ #include <linux/list_sort.h> #include <linux/prime_numbers.h> -#include "../i915_selftest.h" +#include "gem/selftests/mock_context.h" + #include "i915_random.h" +#include "i915_selftest.h" -#include "mock_context.h" #include "mock_drm.h" #include "mock_gem_device.h" @@ -147,7 +148,7 @@ err: static int igt_ppgtt_alloc(void *arg) { struct drm_i915_private *dev_priv = arg; - struct i915_hw_ppgtt *ppgtt; + struct i915_ppgtt *ppgtt; u64 size, last, limit; int err = 0; @@ -156,7 +157,7 @@ static int igt_ppgtt_alloc(void *arg) if (!HAS_PPGTT(dev_priv)) return 0; - ppgtt = __hw_ppgtt_create(dev_priv); + ppgtt = __ppgtt_create(dev_priv); if (IS_ERR(ppgtt)) return PTR_ERR(ppgtt); @@ -208,7 +209,7 @@ static int igt_ppgtt_alloc(void *arg) err_ppgtt_cleanup: mutex_lock(&dev_priv->drm.struct_mutex); - i915_ppgtt_put(ppgtt); + i915_vm_put(&ppgtt->vm); mutex_unlock(&dev_priv->drm.struct_mutex); return err; } @@ -294,9 +295,9 @@ static int lowlevel_hole(struct drm_i915_private *i915, mock_vma.node.size = BIT_ULL(size); mock_vma.node.start = addr; - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); vm->insert_entries(vm, &mock_vma, I915_CACHE_NONE, 0); - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); } count = n; @@ -998,7 +999,7 @@ static int exercise_ppgtt(struct drm_i915_private *dev_priv, unsigned long end_time)) { struct drm_file *file; - struct i915_hw_ppgtt *ppgtt; + struct i915_ppgtt *ppgtt; IGT_TIMEOUT(end_time); int err; @@ -1020,7 +1021,7 @@ static int exercise_ppgtt(struct drm_i915_private *dev_priv, err = func(dev_priv, &ppgtt->vm, 0, ppgtt->vm.total, end_time); - i915_ppgtt_put(ppgtt); + i915_vm_put(&ppgtt->vm); out_unlock: mutex_unlock(&dev_priv->drm.struct_mutex); @@ -1170,7 +1171,7 @@ static int igt_ggtt_page(void *arg) if (err) goto out_unpin; - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); for (n = 0; n < count; n++) { u64 offset = tmp.start + n * PAGE_SIZE; @@ -1217,7 +1218,7 @@ static int igt_ggtt_page(void *arg) kfree(order); out_remove: ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size); - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); drm_mm_remove_node(&tmp); out_unpin: i915_gem_object_unpin_pages(obj); @@ -1232,7 +1233,7 @@ static void track_vma_bind(struct i915_vma *vma) { struct drm_i915_gem_object *obj = vma->obj; - obj->bind_count++; /* track for eviction later */ + atomic_inc(&obj->bind_count); /* track for eviction later */ __i915_gem_object_pin_pages(obj); vma->pages = obj->mm.pages; @@ -1250,7 +1251,6 @@ static int exercise_mock(struct drm_i915_private *i915, { const u64 limit = totalram_pages() << PAGE_SHIFT; struct i915_gem_context *ctx; - struct i915_hw_ppgtt *ppgtt; IGT_TIMEOUT(end_time); int err; @@ -1258,10 +1258,7 @@ static int exercise_mock(struct drm_i915_private *i915, if (!ctx) return -ENOMEM; - ppgtt = ctx->ppgtt; - GEM_BUG_ON(!ppgtt); - - err = func(i915, &ppgtt->vm, 0, min(ppgtt->vm.total, limit), end_time); + err = func(i915, ctx->vm, 0, min(ctx->vm->total, limit), end_time); mock_context_close(ctx); return err; diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_object.c b/drivers/gpu/drm/i915/selftests/i915_gem_object.c deleted file mode 100644 index 971148fbe6f5..000000000000 --- a/drivers/gpu/drm/i915/selftests/i915_gem_object.c +++ /dev/null @@ -1,659 +0,0 @@ -/* - * Copyright © 2016 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#include "../i915_selftest.h" - -#include "mock_gem_device.h" -#include "huge_gem_object.h" - -static int igt_gem_object(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct drm_i915_gem_object *obj; - int err = -ENOMEM; - - /* Basic test to ensure we can create an object */ - - obj = i915_gem_object_create(i915, PAGE_SIZE); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - pr_err("i915_gem_object_create failed, err=%d\n", err); - goto out; - } - - err = 0; - i915_gem_object_put(obj); -out: - return err; -} - -static int igt_phys_object(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct drm_i915_gem_object *obj; - int err; - - /* Create an object and bind it to a contiguous set of physical pages, - * i.e. exercise the i915_gem_object_phys API. - */ - - obj = i915_gem_object_create(i915, PAGE_SIZE); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - pr_err("i915_gem_object_create failed, err=%d\n", err); - goto out; - } - - mutex_lock(&i915->drm.struct_mutex); - err = i915_gem_object_attach_phys(obj, PAGE_SIZE); - mutex_unlock(&i915->drm.struct_mutex); - if (err) { - pr_err("i915_gem_object_attach_phys failed, err=%d\n", err); - goto out_obj; - } - - if (obj->ops != &i915_gem_phys_ops) { - pr_err("i915_gem_object_attach_phys did not create a phys object\n"); - err = -EINVAL; - goto out_obj; - } - - if (!atomic_read(&obj->mm.pages_pin_count)) { - pr_err("i915_gem_object_attach_phys did not pin its phys pages\n"); - err = -EINVAL; - goto out_obj; - } - - /* Make the object dirty so that put_pages must do copy back the data */ - mutex_lock(&i915->drm.struct_mutex); - err = i915_gem_object_set_to_gtt_domain(obj, true); - mutex_unlock(&i915->drm.struct_mutex); - if (err) { - pr_err("i915_gem_object_set_to_gtt_domain failed with err=%d\n", - err); - goto out_obj; - } - -out_obj: - i915_gem_object_put(obj); -out: - return err; -} - -static int igt_gem_huge(void *arg) -{ - const unsigned int nreal = 509; /* just to be awkward */ - struct drm_i915_private *i915 = arg; - struct drm_i915_gem_object *obj; - unsigned int n; - int err; - - /* Basic sanitycheck of our huge fake object allocation */ - - obj = huge_gem_object(i915, - nreal * PAGE_SIZE, - i915->ggtt.vm.total + PAGE_SIZE); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - err = i915_gem_object_pin_pages(obj); - if (err) { - pr_err("Failed to allocate %u pages (%lu total), err=%d\n", - nreal, obj->base.size / PAGE_SIZE, err); - goto out; - } - - for (n = 0; n < obj->base.size / PAGE_SIZE; n++) { - if (i915_gem_object_get_page(obj, n) != - i915_gem_object_get_page(obj, n % nreal)) { - pr_err("Page lookup mismatch at index %u [%u]\n", - n, n % nreal); - err = -EINVAL; - goto out_unpin; - } - } - -out_unpin: - i915_gem_object_unpin_pages(obj); -out: - i915_gem_object_put(obj); - return err; -} - -struct tile { - unsigned int width; - unsigned int height; - unsigned int stride; - unsigned int size; - unsigned int tiling; - unsigned int swizzle; -}; - -static u64 swizzle_bit(unsigned int bit, u64 offset) -{ - return (offset & BIT_ULL(bit)) >> (bit - 6); -} - -static u64 tiled_offset(const struct tile *tile, u64 v) -{ - u64 x, y; - - if (tile->tiling == I915_TILING_NONE) - return v; - - y = div64_u64_rem(v, tile->stride, &x); - v = div64_u64_rem(y, tile->height, &y) * tile->stride * tile->height; - - if (tile->tiling == I915_TILING_X) { - v += y * tile->width; - v += div64_u64_rem(x, tile->width, &x) << tile->size; - v += x; - } else if (tile->width == 128) { - const unsigned int ytile_span = 16; - const unsigned int ytile_height = 512; - - v += y * ytile_span; - v += div64_u64_rem(x, ytile_span, &x) * ytile_height; - v += x; - } else { - const unsigned int ytile_span = 32; - const unsigned int ytile_height = 256; - - v += y * ytile_span; - v += div64_u64_rem(x, ytile_span, &x) * ytile_height; - v += x; - } - - switch (tile->swizzle) { - case I915_BIT_6_SWIZZLE_9: - v ^= swizzle_bit(9, v); - break; - case I915_BIT_6_SWIZZLE_9_10: - v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v); - break; - case I915_BIT_6_SWIZZLE_9_11: - v ^= swizzle_bit(9, v) ^ swizzle_bit(11, v); - break; - case I915_BIT_6_SWIZZLE_9_10_11: - v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v) ^ swizzle_bit(11, v); - break; - } - - return v; -} - -static int check_partial_mapping(struct drm_i915_gem_object *obj, - const struct tile *tile, - unsigned long end_time) -{ - const unsigned int nreal = obj->scratch / PAGE_SIZE; - const unsigned long npages = obj->base.size / PAGE_SIZE; - struct i915_vma *vma; - unsigned long page; - int err; - - if (igt_timeout(end_time, - "%s: timed out before tiling=%d stride=%d\n", - __func__, tile->tiling, tile->stride)) - return -EINTR; - - err = i915_gem_object_set_tiling(obj, tile->tiling, tile->stride); - if (err) { - pr_err("Failed to set tiling mode=%u, stride=%u, err=%d\n", - tile->tiling, tile->stride, err); - return err; - } - - GEM_BUG_ON(i915_gem_object_get_tiling(obj) != tile->tiling); - GEM_BUG_ON(i915_gem_object_get_stride(obj) != tile->stride); - - for_each_prime_number_from(page, 1, npages) { - struct i915_ggtt_view view = - compute_partial_view(obj, page, MIN_CHUNK_PAGES); - u32 __iomem *io; - struct page *p; - unsigned int n; - u64 offset; - u32 *cpu; - - GEM_BUG_ON(view.partial.size > nreal); - cond_resched(); - - err = i915_gem_object_set_to_gtt_domain(obj, true); - if (err) { - pr_err("Failed to flush to GTT write domain; err=%d\n", - err); - return err; - } - - vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE); - if (IS_ERR(vma)) { - pr_err("Failed to pin partial view: offset=%lu; err=%d\n", - page, (int)PTR_ERR(vma)); - return PTR_ERR(vma); - } - - n = page - view.partial.offset; - GEM_BUG_ON(n >= view.partial.size); - - io = i915_vma_pin_iomap(vma); - i915_vma_unpin(vma); - if (IS_ERR(io)) { - pr_err("Failed to iomap partial view: offset=%lu; err=%d\n", - page, (int)PTR_ERR(io)); - return PTR_ERR(io); - } - - iowrite32(page, io + n * PAGE_SIZE/sizeof(*io)); - i915_vma_unpin_iomap(vma); - - offset = tiled_offset(tile, page << PAGE_SHIFT); - if (offset >= obj->base.size) - continue; - - flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU); - - p = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT); - cpu = kmap(p) + offset_in_page(offset); - drm_clflush_virt_range(cpu, sizeof(*cpu)); - if (*cpu != (u32)page) { - pr_err("Partial view for %lu [%u] (offset=%llu, size=%u [%llu, row size %u], fence=%d, tiling=%d, stride=%d) misalignment, expected write to page (%llu + %u [0x%llx]) of 0x%x, found 0x%x\n", - page, n, - view.partial.offset, - view.partial.size, - vma->size >> PAGE_SHIFT, - tile->tiling ? tile_row_pages(obj) : 0, - vma->fence ? vma->fence->id : -1, tile->tiling, tile->stride, - offset >> PAGE_SHIFT, - (unsigned int)offset_in_page(offset), - offset, - (u32)page, *cpu); - err = -EINVAL; - } - *cpu = 0; - drm_clflush_virt_range(cpu, sizeof(*cpu)); - kunmap(p); - if (err) - return err; - - i915_vma_destroy(vma); - } - - return 0; -} - -static int igt_partial_tiling(void *arg) -{ - const unsigned int nreal = 1 << 12; /* largest tile row x2 */ - struct drm_i915_private *i915 = arg; - struct drm_i915_gem_object *obj; - intel_wakeref_t wakeref; - int tiling; - int err; - - /* We want to check the page mapping and fencing of a large object - * mmapped through the GTT. The object we create is larger than can - * possibly be mmaped as a whole, and so we must use partial GGTT vma. - * We then check that a write through each partial GGTT vma ends up - * in the right set of pages within the object, and with the expected - * tiling, which we verify by manual swizzling. - */ - - obj = huge_gem_object(i915, - nreal << PAGE_SHIFT, - (1 + next_prime_number(i915->ggtt.vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - err = i915_gem_object_pin_pages(obj); - if (err) { - pr_err("Failed to allocate %u pages (%lu total), err=%d\n", - nreal, obj->base.size / PAGE_SIZE, err); - goto out; - } - - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); - - if (1) { - IGT_TIMEOUT(end); - struct tile tile; - - tile.height = 1; - tile.width = 1; - tile.size = 0; - tile.stride = 0; - tile.swizzle = I915_BIT_6_SWIZZLE_NONE; - tile.tiling = I915_TILING_NONE; - - err = check_partial_mapping(obj, &tile, end); - if (err && err != -EINTR) - goto out_unlock; - } - - for (tiling = I915_TILING_X; tiling <= I915_TILING_Y; tiling++) { - IGT_TIMEOUT(end); - unsigned int max_pitch; - unsigned int pitch; - struct tile tile; - - if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) - /* - * The swizzling pattern is actually unknown as it - * varies based on physical address of each page. - * See i915_gem_detect_bit_6_swizzle(). - */ - break; - - tile.tiling = tiling; - switch (tiling) { - case I915_TILING_X: - tile.swizzle = i915->mm.bit_6_swizzle_x; - break; - case I915_TILING_Y: - tile.swizzle = i915->mm.bit_6_swizzle_y; - break; - } - - GEM_BUG_ON(tile.swizzle == I915_BIT_6_SWIZZLE_UNKNOWN); - if (tile.swizzle == I915_BIT_6_SWIZZLE_9_17 || - tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17) - continue; - - if (INTEL_GEN(i915) <= 2) { - tile.height = 16; - tile.width = 128; - tile.size = 11; - } else if (tile.tiling == I915_TILING_Y && - HAS_128_BYTE_Y_TILING(i915)) { - tile.height = 32; - tile.width = 128; - tile.size = 12; - } else { - tile.height = 8; - tile.width = 512; - tile.size = 12; - } - - if (INTEL_GEN(i915) < 4) - max_pitch = 8192 / tile.width; - else if (INTEL_GEN(i915) < 7) - max_pitch = 128 * I965_FENCE_MAX_PITCH_VAL / tile.width; - else - max_pitch = 128 * GEN7_FENCE_MAX_PITCH_VAL / tile.width; - - for (pitch = max_pitch; pitch; pitch >>= 1) { - tile.stride = tile.width * pitch; - err = check_partial_mapping(obj, &tile, end); - if (err == -EINTR) - goto next_tiling; - if (err) - goto out_unlock; - - if (pitch > 2 && INTEL_GEN(i915) >= 4) { - tile.stride = tile.width * (pitch - 1); - err = check_partial_mapping(obj, &tile, end); - if (err == -EINTR) - goto next_tiling; - if (err) - goto out_unlock; - } - - if (pitch < max_pitch && INTEL_GEN(i915) >= 4) { - tile.stride = tile.width * (pitch + 1); - err = check_partial_mapping(obj, &tile, end); - if (err == -EINTR) - goto next_tiling; - if (err) - goto out_unlock; - } - } - - if (INTEL_GEN(i915) >= 4) { - for_each_prime_number(pitch, max_pitch) { - tile.stride = tile.width * pitch; - err = check_partial_mapping(obj, &tile, end); - if (err == -EINTR) - goto next_tiling; - if (err) - goto out_unlock; - } - } - -next_tiling: ; - } - -out_unlock: - intel_runtime_pm_put(i915, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - i915_gem_object_unpin_pages(obj); -out: - i915_gem_object_put(obj); - return err; -} - -static int make_obj_busy(struct drm_i915_gem_object *obj) -{ - struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_request *rq; - struct i915_vma *vma; - int err; - - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); - if (IS_ERR(vma)) - return PTR_ERR(vma); - - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) - return err; - - rq = i915_request_alloc(i915->engine[RCS0], i915->kernel_context); - if (IS_ERR(rq)) { - i915_vma_unpin(vma); - return PTR_ERR(rq); - } - - err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); - - i915_request_add(rq); - - __i915_gem_object_release_unless_active(obj); - i915_vma_unpin(vma); - - return err; -} - -static bool assert_mmap_offset(struct drm_i915_private *i915, - unsigned long size, - int expected) -{ - struct drm_i915_gem_object *obj; - int err; - - obj = i915_gem_object_create_internal(i915, size); - if (IS_ERR(obj)) - return PTR_ERR(obj); - - err = i915_gem_object_create_mmap_offset(obj); - i915_gem_object_put(obj); - - return err == expected; -} - -static void disable_retire_worker(struct drm_i915_private *i915) -{ - i915_gem_shrinker_unregister(i915); - - mutex_lock(&i915->drm.struct_mutex); - if (!i915->gt.active_requests++) { - intel_wakeref_t wakeref; - - with_intel_runtime_pm(i915, wakeref) - i915_gem_unpark(i915); - } - mutex_unlock(&i915->drm.struct_mutex); - - cancel_delayed_work_sync(&i915->gt.retire_work); - cancel_delayed_work_sync(&i915->gt.idle_work); -} - -static int igt_mmap_offset_exhaustion(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct drm_mm *mm = &i915->drm.vma_offset_manager->vm_addr_space_mm; - struct drm_i915_gem_object *obj; - struct drm_mm_node resv, *hole; - u64 hole_start, hole_end; - int loop, err; - - /* Disable background reaper */ - disable_retire_worker(i915); - GEM_BUG_ON(!i915->gt.awake); - - /* Trim the device mmap space to only a page */ - memset(&resv, 0, sizeof(resv)); - drm_mm_for_each_hole(hole, mm, hole_start, hole_end) { - resv.start = hole_start; - resv.size = hole_end - hole_start - 1; /* PAGE_SIZE units */ - err = drm_mm_reserve_node(mm, &resv); - if (err) { - pr_err("Failed to trim VMA manager, err=%d\n", err); - goto out_park; - } - break; - } - - /* Just fits! */ - if (!assert_mmap_offset(i915, PAGE_SIZE, 0)) { - pr_err("Unable to insert object into single page hole\n"); - err = -EINVAL; - goto out; - } - - /* Too large */ - if (!assert_mmap_offset(i915, 2*PAGE_SIZE, -ENOSPC)) { - pr_err("Unexpectedly succeeded in inserting too large object into single page hole\n"); - err = -EINVAL; - goto out; - } - - /* Fill the hole, further allocation attempts should then fail */ - obj = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto out; - } - - err = i915_gem_object_create_mmap_offset(obj); - if (err) { - pr_err("Unable to insert object into reclaimed hole\n"); - goto err_obj; - } - - if (!assert_mmap_offset(i915, PAGE_SIZE, -ENOSPC)) { - pr_err("Unexpectedly succeeded in inserting object into no holes!\n"); - err = -EINVAL; - goto err_obj; - } - - i915_gem_object_put(obj); - - /* Now fill with busy dead objects that we expect to reap */ - for (loop = 0; loop < 3; loop++) { - intel_wakeref_t wakeref; - - if (i915_terminally_wedged(i915)) - break; - - obj = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto out; - } - - err = 0; - mutex_lock(&i915->drm.struct_mutex); - with_intel_runtime_pm(i915, wakeref) - err = make_obj_busy(obj); - mutex_unlock(&i915->drm.struct_mutex); - if (err) { - pr_err("[loop %d] Failed to busy the object\n", loop); - goto err_obj; - } - - /* NB we rely on the _active_ reference to access obj now */ - GEM_BUG_ON(!i915_gem_object_is_active(obj)); - err = i915_gem_object_create_mmap_offset(obj); - if (err) { - pr_err("[loop %d] i915_gem_object_create_mmap_offset failed with err=%d\n", - loop, err); - goto out; - } - } - -out: - drm_mm_remove_node(&resv); -out_park: - mutex_lock(&i915->drm.struct_mutex); - if (--i915->gt.active_requests) - queue_delayed_work(i915->wq, &i915->gt.retire_work, 0); - else - queue_delayed_work(i915->wq, &i915->gt.idle_work, 0); - mutex_unlock(&i915->drm.struct_mutex); - i915_gem_shrinker_register(i915); - return err; -err_obj: - i915_gem_object_put(obj); - goto out; -} - -int i915_gem_object_mock_selftests(void) -{ - static const struct i915_subtest tests[] = { - SUBTEST(igt_gem_object), - SUBTEST(igt_phys_object), - }; - struct drm_i915_private *i915; - int err; - - i915 = mock_gem_device(); - if (!i915) - return -ENOMEM; - - err = i915_subtests(tests, i915); - - drm_dev_put(&i915->drm); - return err; -} - -int i915_gem_object_live_selftests(struct drm_i915_private *i915) -{ - static const struct i915_subtest tests[] = { - SUBTEST(igt_gem_huge), - SUBTEST(igt_partial_tiling), - SUBTEST(igt_mmap_offset_exhaustion), - }; - - return i915_subtests(tests, i915); -} diff --git a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h index 6d766925ad04..d5dc4427d664 100644 --- a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h @@ -16,13 +16,18 @@ selftest(timelines, i915_timeline_live_selftests) selftest(requests, i915_request_live_selftests) selftest(active, i915_active_live_selftests) selftest(objects, i915_gem_object_live_selftests) +selftest(mman, i915_gem_mman_live_selftests) selftest(dmabuf, i915_gem_dmabuf_live_selftests) +selftest(vma, i915_vma_live_selftests) selftest(coherency, i915_gem_coherency_live_selftests) selftest(gtt, i915_gem_gtt_live_selftests) selftest(gem, i915_gem_live_selftests) selftest(evict, i915_gem_evict_live_selftests) selftest(hugepages, i915_gem_huge_page_live_selftests) selftest(contexts, i915_gem_context_live_selftests) +selftest(blt, i915_gem_object_blt_live_selftests) +selftest(client, i915_gem_client_blt_live_selftests) +selftest(reset, intel_reset_live_selftests) selftest(hangcheck, intel_hangcheck_live_selftests) selftest(execlists, intel_execlists_live_selftests) selftest(guc, intel_guc_live_selftest) diff --git a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h index 88e5ab586337..510eb176bb2c 100644 --- a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h @@ -18,6 +18,7 @@ selftest(engine, intel_engine_cs_mock_selftests) selftest(timelines, i915_timeline_mock_selftests) selftest(requests, i915_request_mock_selftests) selftest(objects, i915_gem_object_mock_selftests) +selftest(phys, i915_gem_phys_mock_selftests) selftest(dmabuf, i915_gem_dmabuf_mock_selftests) selftest(vma, i915_vma_mock_selftests) selftest(evict, i915_gem_evict_mock_selftests) diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c index e6ffe2240126..298bb7116c51 100644 --- a/drivers/gpu/drm/i915/selftests/i915_request.c +++ b/drivers/gpu/drm/i915/selftests/i915_request.c @@ -24,12 +24,14 @@ #include <linux/prime_numbers.h> -#include "../i915_selftest.h" +#include "gem/i915_gem_pm.h" +#include "gem/selftests/mock_context.h" + #include "i915_random.h" +#include "i915_selftest.h" #include "igt_live_test.h" #include "lib_sw_fence.h" -#include "mock_context.h" #include "mock_drm.h" #include "mock_gem_device.h" @@ -72,12 +74,12 @@ static int igt_wait_request(void *arg) goto out_unlock; } - if (i915_request_wait(request, I915_WAIT_LOCKED, 0) != -ETIME) { + if (i915_request_wait(request, 0, 0) != -ETIME) { pr_err("request wait (busy query) succeeded (expected timeout before submit!)\n"); goto out_unlock; } - if (i915_request_wait(request, I915_WAIT_LOCKED, T) != -ETIME) { + if (i915_request_wait(request, 0, T) != -ETIME) { pr_err("request wait succeeded (expected timeout before submit!)\n"); goto out_unlock; } @@ -89,7 +91,7 @@ static int igt_wait_request(void *arg) i915_request_add(request); - if (i915_request_wait(request, I915_WAIT_LOCKED, 0) != -ETIME) { + if (i915_request_wait(request, 0, 0) != -ETIME) { pr_err("request wait (busy query) succeeded (expected timeout after submit!)\n"); goto out_unlock; } @@ -99,12 +101,12 @@ static int igt_wait_request(void *arg) goto out_unlock; } - if (i915_request_wait(request, I915_WAIT_LOCKED, T / 2) != -ETIME) { + if (i915_request_wait(request, 0, T / 2) != -ETIME) { pr_err("request wait succeeded (expected timeout!)\n"); goto out_unlock; } - if (i915_request_wait(request, I915_WAIT_LOCKED, T) == -ETIME) { + if (i915_request_wait(request, 0, T) == -ETIME) { pr_err("request wait timed out!\n"); goto out_unlock; } @@ -114,7 +116,7 @@ static int igt_wait_request(void *arg) goto out_unlock; } - if (i915_request_wait(request, I915_WAIT_LOCKED, T) == -ETIME) { + if (i915_request_wait(request, 0, T) == -ETIME) { pr_err("request wait timed out when already complete!\n"); goto out_unlock; } @@ -267,7 +269,7 @@ static struct i915_request * __live_request_alloc(struct i915_gem_context *ctx, struct intel_engine_cs *engine) { - return i915_request_alloc(engine, ctx); + return igt_request_alloc(ctx, engine); } static int __igt_breadcrumbs_smoketest(void *arg) @@ -512,7 +514,7 @@ int i915_request_mock_selftests(void) if (!i915) return -ENOMEM; - with_intel_runtime_pm(i915, wakeref) + with_intel_runtime_pm(&i915->runtime_pm, wakeref) err = i915_subtests(tests, i915); drm_dev_put(&i915->drm); @@ -535,7 +537,7 @@ static int live_nop_request(void *arg) */ mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); for_each_engine(engine, i915, id) { struct i915_request *request = NULL; @@ -551,8 +553,7 @@ static int live_nop_request(void *arg) times[1] = ktime_get_raw(); for (n = 0; n < prime; n++) { - request = i915_request_alloc(engine, - i915->kernel_context); + request = i915_request_create(engine->kernel_context); if (IS_ERR(request)) { err = PTR_ERR(request); goto out_unlock; @@ -573,9 +574,7 @@ static int live_nop_request(void *arg) i915_request_add(request); } - i915_request_wait(request, - I915_WAIT_LOCKED, - MAX_SCHEDULE_TIMEOUT); + i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT); times[1] = ktime_sub(ktime_get_raw(), times[1]); if (prime == 1) @@ -596,7 +595,7 @@ static int live_nop_request(void *arg) } out_unlock: - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; } @@ -649,7 +648,7 @@ empty_request(struct intel_engine_cs *engine, struct i915_request *request; int err; - request = i915_request_alloc(engine, engine->i915->kernel_context); + request = i915_request_create(engine->kernel_context); if (IS_ERR(request)) return request; @@ -681,7 +680,7 @@ static int live_empty_request(void *arg) */ mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); batch = empty_batch(i915); if (IS_ERR(batch)) { @@ -705,9 +704,7 @@ static int live_empty_request(void *arg) err = PTR_ERR(request); goto out_batch; } - i915_request_wait(request, - I915_WAIT_LOCKED, - MAX_SCHEDULE_TIMEOUT); + i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT); for_each_prime_number_from(prime, 1, 8192) { times[1] = ktime_get_raw(); @@ -719,9 +716,7 @@ static int live_empty_request(void *arg) goto out_batch; } } - i915_request_wait(request, - I915_WAIT_LOCKED, - MAX_SCHEDULE_TIMEOUT); + i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT); times[1] = ktime_sub(ktime_get_raw(), times[1]); if (prime == 1) @@ -745,7 +740,7 @@ out_batch: i915_vma_unpin(batch); i915_vma_put(batch); out_unlock: - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; } @@ -753,8 +748,7 @@ out_unlock: static struct i915_vma *recursive_batch(struct drm_i915_private *i915) { struct i915_gem_context *ctx = i915->kernel_context; - struct i915_address_space *vm = - ctx->ppgtt ? &ctx->ppgtt->vm : &i915->ggtt.vm; + struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm; struct drm_i915_gem_object *obj; const int gen = INTEL_GEN(i915); struct i915_vma *vma; @@ -839,7 +833,7 @@ static int live_all_engines(void *arg) */ mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); err = igt_live_test_begin(&t, i915, __func__, ""); if (err) @@ -853,7 +847,7 @@ static int live_all_engines(void *arg) } for_each_engine(engine, i915, id) { - request[id] = i915_request_alloc(engine, i915->kernel_context); + request[id] = i915_request_create(engine->kernel_context); if (IS_ERR(request[id])) { err = PTR_ERR(request[id]); pr_err("%s: Request allocation failed with err=%d\n", @@ -868,12 +862,9 @@ static int live_all_engines(void *arg) GEM_BUG_ON(err); request[id]->batch = batch; - if (!i915_gem_object_has_active_reference(batch->obj)) { - i915_gem_object_get(batch->obj); - i915_gem_object_set_active_reference(batch->obj); - } - + i915_vma_lock(batch); err = i915_vma_move_to_active(batch, request[id], 0); + i915_vma_unlock(batch); GEM_BUG_ON(err); i915_request_get(request[id]); @@ -898,8 +889,7 @@ static int live_all_engines(void *arg) for_each_engine(engine, i915, id) { long timeout; - timeout = i915_request_wait(request[id], - I915_WAIT_LOCKED, + timeout = i915_request_wait(request[id], 0, MAX_SCHEDULE_TIMEOUT); if (timeout < 0) { err = timeout; @@ -922,7 +912,7 @@ out_request: i915_vma_unpin(batch); i915_vma_put(batch); out_unlock: - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; } @@ -945,7 +935,7 @@ static int live_sequential_engines(void *arg) */ mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); err = igt_live_test_begin(&t, i915, __func__, ""); if (err) @@ -962,7 +952,7 @@ static int live_sequential_engines(void *arg) goto out_unlock; } - request[id] = i915_request_alloc(engine, i915->kernel_context); + request[id] = i915_request_create(engine->kernel_context); if (IS_ERR(request[id])) { err = PTR_ERR(request[id]); pr_err("%s: Request allocation failed for %s with err=%d\n", @@ -988,12 +978,11 @@ static int live_sequential_engines(void *arg) GEM_BUG_ON(err); request[id]->batch = batch; + i915_vma_lock(batch); err = i915_vma_move_to_active(batch, request[id], 0); + i915_vma_unlock(batch); GEM_BUG_ON(err); - i915_gem_object_set_active_reference(batch->obj); - i915_vma_get(batch); - i915_request_get(request[id]); i915_request_add(request[id]); @@ -1017,8 +1006,7 @@ static int live_sequential_engines(void *arg) goto out_request; } - timeout = i915_request_wait(request[id], - I915_WAIT_LOCKED, + timeout = i915_request_wait(request[id], 0, MAX_SCHEDULE_TIMEOUT); if (timeout < 0) { err = timeout; @@ -1052,7 +1040,7 @@ out_request: i915_request_put(request[id]); } out_unlock: - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; } @@ -1075,7 +1063,7 @@ max_batches(struct i915_gem_context *ctx, struct intel_engine_cs *engine) if (HAS_EXECLISTS(ctx->i915)) return INT_MAX; - rq = i915_request_alloc(engine, ctx); + rq = igt_request_alloc(ctx, engine); if (IS_ERR(rq)) { ret = PTR_ERR(rq); } else { @@ -1117,7 +1105,7 @@ static int live_breadcrumbs_smoketest(void *arg) * On real hardware this time. */ - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); file = mock_file(i915); if (IS_ERR(file)) { @@ -1224,7 +1212,7 @@ out_threads: out_file: mock_file_free(i915, file); out_rpm: - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); return ret; } diff --git a/drivers/gpu/drm/i915/selftests/i915_timeline.c b/drivers/gpu/drm/i915/selftests/i915_timeline.c index bd96afcadfe7..76d3977f1d4b 100644 --- a/drivers/gpu/drm/i915/selftests/i915_timeline.c +++ b/drivers/gpu/drm/i915/selftests/i915_timeline.c @@ -6,8 +6,10 @@ #include <linux/prime_numbers.h> -#include "../i915_selftest.h" +#include "gem/i915_gem_pm.h" + #include "i915_random.h" +#include "i915_selftest.h" #include "igt_flush_test.h" #include "mock_gem_device.h" @@ -454,7 +456,7 @@ tl_write(struct i915_timeline *tl, struct intel_engine_cs *engine, u32 value) goto out; } - rq = i915_request_alloc(engine, engine->i915->kernel_context); + rq = i915_request_create(engine->kernel_context); if (IS_ERR(rq)) goto out_unpin; @@ -513,7 +515,7 @@ static int live_hwsp_engine(void *arg) return -ENOMEM; mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); count = 0; for_each_engine(engine, i915, id) { @@ -556,7 +558,7 @@ out: i915_timeline_put(tl); } - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); mutex_unlock(&i915->drm.struct_mutex); kvfree(timelines); @@ -589,7 +591,7 @@ static int live_hwsp_alternate(void *arg) return -ENOMEM; mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); count = 0; for (n = 0; n < NUM_TIMELINES; n++) { @@ -632,7 +634,7 @@ out: i915_timeline_put(tl); } - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); mutex_unlock(&i915->drm.struct_mutex); kvfree(timelines); @@ -656,7 +658,7 @@ static int live_hwsp_wrap(void *arg) */ mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); tl = i915_timeline_create(i915, NULL); if (IS_ERR(tl)) { @@ -678,7 +680,7 @@ static int live_hwsp_wrap(void *arg) if (!intel_engine_can_store_dword(engine)) continue; - rq = i915_request_alloc(engine, i915->kernel_context); + rq = i915_request_create(engine->kernel_context); if (IS_ERR(rq)) { err = PTR_ERR(rq); goto out; @@ -722,7 +724,7 @@ static int live_hwsp_wrap(void *arg) i915_request_add(rq); - if (i915_request_wait(rq, I915_WAIT_LOCKED, HZ / 5) < 0) { + if (i915_request_wait(rq, 0, HZ / 5) < 0) { pr_err("Wait for timeline writes timed out!\n"); err = -EIO; goto out; @@ -747,7 +749,7 @@ out: out_free: i915_timeline_put(tl); out_rpm: - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; @@ -769,7 +771,7 @@ static int live_hwsp_recycle(void *arg) */ mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); count = 0; for_each_engine(engine, i915, id) { @@ -795,9 +797,7 @@ static int live_hwsp_recycle(void *arg) goto out; } - if (i915_request_wait(rq, - I915_WAIT_LOCKED, - HZ / 5) < 0) { + if (i915_request_wait(rq, 0, HZ / 5) < 0) { pr_err("Wait for timeline writes timed out!\n"); i915_timeline_put(tl); err = -EIO; @@ -823,7 +823,7 @@ static int live_hwsp_recycle(void *arg) out: if (igt_flush_test(i915, I915_WAIT_LOCKED)) err = -EIO; - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); mutex_unlock(&i915->drm.struct_mutex); return err; diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c index fc594b030f5a..fbc79b14823a 100644 --- a/drivers/gpu/drm/i915/selftests/i915_vma.c +++ b/drivers/gpu/drm/i915/selftests/i915_vma.c @@ -24,10 +24,12 @@ #include <linux/prime_numbers.h> -#include "../i915_selftest.h" +#include "gem/selftests/mock_context.h" + +#include "i915_scatterlist.h" +#include "i915_selftest.h" #include "mock_gem_device.h" -#include "mock_context.h" #include "mock_gtt.h" static bool assert_vma(struct i915_vma *vma, @@ -36,7 +38,7 @@ static bool assert_vma(struct i915_vma *vma, { bool ok = true; - if (vma->vm != &ctx->ppgtt->vm) { + if (vma->vm != ctx->vm) { pr_err("VMA created with wrong VM\n"); ok = false; } @@ -59,7 +61,7 @@ static bool assert_vma(struct i915_vma *vma, static struct i915_vma * checked_vma_instance(struct drm_i915_gem_object *obj, struct i915_address_space *vm, - struct i915_ggtt_view *view) + const struct i915_ggtt_view *view) { struct i915_vma *vma; bool ok = true; @@ -111,7 +113,7 @@ static int create_vmas(struct drm_i915_private *i915, list_for_each_entry(obj, objects, st_link) { for (pinned = 0; pinned <= 1; pinned++) { list_for_each_entry(ctx, contexts, link) { - struct i915_address_space *vm = &ctx->ppgtt->vm; + struct i915_address_space *vm = ctx->vm; struct i915_vma *vma; int err; @@ -397,18 +399,79 @@ assert_rotated(struct drm_i915_gem_object *obj, return sg; } -static unsigned int rotated_size(const struct intel_rotation_plane_info *a, - const struct intel_rotation_plane_info *b) +static unsigned long remapped_index(const struct intel_remapped_info *r, + unsigned int n, + unsigned int x, + unsigned int y) +{ + return (r->plane[n].stride * y + + r->plane[n].offset + x); +} + +static struct scatterlist * +assert_remapped(struct drm_i915_gem_object *obj, + const struct intel_remapped_info *r, unsigned int n, + struct scatterlist *sg) +{ + unsigned int x, y; + unsigned int left = 0; + unsigned int offset; + + for (y = 0; y < r->plane[n].height; y++) { + for (x = 0; x < r->plane[n].width; x++) { + unsigned long src_idx; + dma_addr_t src; + + if (!sg) { + pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n", + n, x, y); + return ERR_PTR(-EINVAL); + } + if (!left) { + offset = 0; + left = sg_dma_len(sg); + } + + src_idx = remapped_index(r, n, x, y); + src = i915_gem_object_get_dma_address(obj, src_idx); + + if (left < PAGE_SIZE || left & (PAGE_SIZE-1)) { + pr_err("Invalid sg.length, found %d, expected %lu for remapped page (%d, %d) [src index %lu]\n", + sg_dma_len(sg), PAGE_SIZE, + x, y, src_idx); + return ERR_PTR(-EINVAL); + } + + if (sg_dma_address(sg) + offset != src) { + pr_err("Invalid address for remapped page (%d, %d) [src index %lu]\n", + x, y, src_idx); + return ERR_PTR(-EINVAL); + } + + left -= PAGE_SIZE; + offset += PAGE_SIZE; + + + if (!left) + sg = sg_next(sg); + } + } + + return sg; +} + +static unsigned int rotated_size(const struct intel_remapped_plane_info *a, + const struct intel_remapped_plane_info *b) { return a->width * a->height + b->width * b->height; } -static int igt_vma_rotate(void *arg) +static int igt_vma_rotate_remap(void *arg) { struct i915_ggtt *ggtt = arg; struct i915_address_space *vm = &ggtt->vm; struct drm_i915_gem_object *obj; - const struct intel_rotation_plane_info planes[] = { + const struct intel_remapped_plane_info planes[] = { { .width = 1, .height = 1, .stride = 1 }, { .width = 2, .height = 2, .stride = 2 }, { .width = 4, .height = 4, .stride = 4 }, @@ -426,6 +489,11 @@ static int igt_vma_rotate(void *arg) { .width = 6, .height = 4, .stride = 6 }, { } }, *a, *b; + enum i915_ggtt_view_type types[] = { + I915_GGTT_VIEW_ROTATED, + I915_GGTT_VIEW_REMAPPED, + 0, + }, *t; const unsigned int max_pages = 64; int err = -ENOMEM; @@ -437,6 +505,7 @@ static int igt_vma_rotate(void *arg) if (IS_ERR(obj)) goto out; + for (t = types; *t; t++) { for (a = planes; a->width; a++) { for (b = planes + ARRAY_SIZE(planes); b-- != planes; ) { struct i915_ggtt_view view; @@ -447,7 +516,7 @@ static int igt_vma_rotate(void *arg) GEM_BUG_ON(max_offset > max_pages); max_offset = max_pages - max_offset; - view.type = I915_GGTT_VIEW_ROTATED; + view.type = *t; view.rotated.plane[0] = *a; view.rotated.plane[1] = *b; @@ -468,14 +537,23 @@ static int igt_vma_rotate(void *arg) goto out_object; } - if (vma->size != rotated_size(a, b) * PAGE_SIZE) { + if (view.type == I915_GGTT_VIEW_ROTATED && + vma->size != rotated_size(a, b) * PAGE_SIZE) { + pr_err("VMA is wrong size, expected %lu, found %llu\n", + PAGE_SIZE * rotated_size(a, b), vma->size); + err = -EINVAL; + goto out_object; + } + + if (view.type == I915_GGTT_VIEW_REMAPPED && + vma->size > rotated_size(a, b) * PAGE_SIZE) { pr_err("VMA is wrong size, expected %lu, found %llu\n", PAGE_SIZE * rotated_size(a, b), vma->size); err = -EINVAL; goto out_object; } - if (vma->pages->nents != rotated_size(a, b)) { + if (vma->pages->nents > rotated_size(a, b)) { pr_err("sg table is wrong sizeo, expected %u, found %u nents\n", rotated_size(a, b), vma->pages->nents); err = -EINVAL; @@ -497,9 +575,14 @@ static int igt_vma_rotate(void *arg) sg = vma->pages->sgl; for (n = 0; n < ARRAY_SIZE(view.rotated.plane); n++) { - sg = assert_rotated(obj, &view.rotated, n, sg); + if (view.type == I915_GGTT_VIEW_ROTATED) + sg = assert_rotated(obj, &view.rotated, n, sg); + else + sg = assert_remapped(obj, &view.remapped, n, sg); if (IS_ERR(sg)) { - pr_err("Inconsistent VMA pages for plane %d: [(%d, %d, %d, %d), (%d, %d, %d, %d)]\n", n, + pr_err("Inconsistent %s VMA pages for plane %d: [(%d, %d, %d, %d), (%d, %d, %d, %d)]\n", + view.type == I915_GGTT_VIEW_ROTATED ? + "rotated" : "remapped", n, view.rotated.plane[0].width, view.rotated.plane[0].height, view.rotated.plane[0].stride, @@ -518,6 +601,7 @@ static int igt_vma_rotate(void *arg) } } } + } out_object: i915_gem_object_put(obj); @@ -721,7 +805,7 @@ int i915_vma_mock_selftests(void) static const struct i915_subtest tests[] = { SUBTEST(igt_vma_create), SUBTEST(igt_vma_pin1), - SUBTEST(igt_vma_rotate), + SUBTEST(igt_vma_rotate_remap), SUBTEST(igt_vma_partial), }; struct drm_i915_private *i915; @@ -752,3 +836,147 @@ out_put: drm_dev_put(&i915->drm); return err; } + +static int igt_vma_remapped_gtt(void *arg) +{ + struct drm_i915_private *i915 = arg; + const struct intel_remapped_plane_info planes[] = { + { .width = 1, .height = 1, .stride = 1 }, + { .width = 2, .height = 2, .stride = 2 }, + { .width = 4, .height = 4, .stride = 4 }, + { .width = 8, .height = 8, .stride = 8 }, + + { .width = 3, .height = 5, .stride = 3 }, + { .width = 3, .height = 5, .stride = 4 }, + { .width = 3, .height = 5, .stride = 5 }, + + { .width = 5, .height = 3, .stride = 5 }, + { .width = 5, .height = 3, .stride = 7 }, + { .width = 5, .height = 3, .stride = 9 }, + + { .width = 4, .height = 6, .stride = 6 }, + { .width = 6, .height = 4, .stride = 6 }, + { } + }, *p; + enum i915_ggtt_view_type types[] = { + I915_GGTT_VIEW_ROTATED, + I915_GGTT_VIEW_REMAPPED, + 0, + }, *t; + struct drm_i915_gem_object *obj; + intel_wakeref_t wakeref; + int err = 0; + + obj = i915_gem_object_create_internal(i915, 10 * 10 * PAGE_SIZE); + if (IS_ERR(obj)) + return PTR_ERR(obj); + + mutex_lock(&i915->drm.struct_mutex); + + wakeref = intel_runtime_pm_get(&i915->runtime_pm); + + for (t = types; *t; t++) { + for (p = planes; p->width; p++) { + struct i915_ggtt_view view = { + .type = *t, + .rotated.plane[0] = *p, + }; + struct i915_vma *vma; + u32 __iomem *map; + unsigned int x, y; + int err; + + i915_gem_object_lock(obj); + err = i915_gem_object_set_to_gtt_domain(obj, true); + i915_gem_object_unlock(obj); + if (err) + goto out; + + vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + goto out; + } + + GEM_BUG_ON(vma->ggtt_view.type != *t); + + map = i915_vma_pin_iomap(vma); + i915_vma_unpin(vma); + if (IS_ERR(map)) { + err = PTR_ERR(map); + goto out; + } + + for (y = 0 ; y < p->height; y++) { + for (x = 0 ; x < p->width; x++) { + unsigned int offset; + u32 val = y << 16 | x; + + if (*t == I915_GGTT_VIEW_ROTATED) + offset = (x * p->height + y) * PAGE_SIZE; + else + offset = (y * p->width + x) * PAGE_SIZE; + + iowrite32(val, &map[offset / sizeof(*map)]); + } + } + + i915_vma_unpin_iomap(vma); + + vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + goto out; + } + + GEM_BUG_ON(vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL); + + map = i915_vma_pin_iomap(vma); + i915_vma_unpin(vma); + if (IS_ERR(map)) { + err = PTR_ERR(map); + goto out; + } + + for (y = 0 ; y < p->height; y++) { + for (x = 0 ; x < p->width; x++) { + unsigned int offset, src_idx; + u32 exp = y << 16 | x; + u32 val; + + if (*t == I915_GGTT_VIEW_ROTATED) + src_idx = rotated_index(&view.rotated, 0, x, y); + else + src_idx = remapped_index(&view.remapped, 0, x, y); + offset = src_idx * PAGE_SIZE; + + val = ioread32(&map[offset / sizeof(*map)]); + if (val != exp) { + pr_err("%s VMA write test failed, expected 0x%x, found 0x%x\n", + *t == I915_GGTT_VIEW_ROTATED ? "Rotated" : "Remapped", + val, exp); + i915_vma_unpin_iomap(vma); + goto out; + } + } + } + i915_vma_unpin_iomap(vma); + } + } + +out: + intel_runtime_pm_put(&i915->runtime_pm, wakeref); + mutex_unlock(&i915->drm.struct_mutex); + i915_gem_object_put(obj); + + return err; +} + +int i915_vma_live_selftests(struct drm_i915_private *i915) +{ + static const struct i915_subtest tests[] = { + SUBTEST(igt_vma_remapped_gtt), + }; + + return i915_subtests(tests, i915); +} diff --git a/drivers/gpu/drm/i915/selftests/igt_atomic.h b/drivers/gpu/drm/i915/selftests/igt_atomic.h new file mode 100644 index 000000000000..93ec89f487ec --- /dev/null +++ b/drivers/gpu/drm/i915/selftests/igt_atomic.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2018 Intel Corporation + */ + +#ifndef IGT_ATOMIC_H +#define IGT_ATOMIC_H + +#include <linux/preempt.h> +#include <linux/bottom_half.h> +#include <linux/irqflags.h> + +static void __preempt_begin(void) +{ + preempt_disable(); +} + +static void __preempt_end(void) +{ + preempt_enable(); +} + +static void __softirq_begin(void) +{ + local_bh_disable(); +} + +static void __softirq_end(void) +{ + local_bh_enable(); +} + +static void __hardirq_begin(void) +{ + local_irq_disable(); +} + +static void __hardirq_end(void) +{ + local_irq_enable(); +} + +struct igt_atomic_section { + const char *name; + void (*critical_section_begin)(void); + void (*critical_section_end)(void); +}; + +static const struct igt_atomic_section igt_atomic_phases[] = { + { "preempt", __preempt_begin, __preempt_end }, + { "softirq", __softirq_begin, __softirq_end }, + { "hardirq", __hardirq_begin, __hardirq_end }, + { } +}; + +#endif /* IGT_ATOMIC_H */ diff --git a/drivers/gpu/drm/i915/selftests/igt_flush_test.c b/drivers/gpu/drm/i915/selftests/igt_flush_test.c index 94aee4071a66..5bfd1b2626a2 100644 --- a/drivers/gpu/drm/i915/selftests/igt_flush_test.c +++ b/drivers/gpu/drm/i915/selftests/igt_flush_test.c @@ -4,30 +4,38 @@ * Copyright © 2018 Intel Corporation */ -#include "../i915_drv.h" +#include "gem/i915_gem_context.h" + +#include "i915_drv.h" +#include "i915_selftest.h" -#include "../i915_selftest.h" #include "igt_flush_test.h" int igt_flush_test(struct drm_i915_private *i915, unsigned int flags) { + int ret = i915_terminally_wedged(i915) ? -EIO : 0; + int repeat = !!(flags & I915_WAIT_LOCKED); + cond_resched(); - if (flags & I915_WAIT_LOCKED && - i915_gem_switch_to_kernel_context(i915, i915->gt.active_engines)) { - pr_err("Failed to switch back to kernel context; declaring wedged\n"); - i915_gem_set_wedged(i915); - } + do { + if (i915_gem_wait_for_idle(i915, flags, HZ / 5) == -ETIME) { + pr_err("%pS timed out, cancelling all further testing.\n", + __builtin_return_address(0)); - if (i915_gem_wait_for_idle(i915, flags, HZ / 5) == -ETIME) { - pr_err("%pS timed out, cancelling all further testing.\n", - __builtin_return_address(0)); + GEM_TRACE("%pS timed out.\n", + __builtin_return_address(0)); + GEM_TRACE_DUMP(); - GEM_TRACE("%pS timed out.\n", __builtin_return_address(0)); - GEM_TRACE_DUMP(); + i915_gem_set_wedged(i915); + repeat = 0; + ret = -EIO; + } - i915_gem_set_wedged(i915); - } + /* Ensure we also flush after wedging. */ + if (flags & I915_WAIT_LOCKED) + i915_retire_requests(i915); + } while (repeat--); - return i915_terminally_wedged(i915); + return ret; } diff --git a/drivers/gpu/drm/i915/selftests/igt_reset.c b/drivers/gpu/drm/i915/selftests/igt_reset.c index 208a966da8ca..587df6fd4ffe 100644 --- a/drivers/gpu/drm/i915/selftests/igt_reset.c +++ b/drivers/gpu/drm/i915/selftests/igt_reset.c @@ -6,8 +6,9 @@ #include "igt_reset.h" +#include "gt/intel_engine.h" + #include "../i915_drv.h" -#include "../intel_ringbuffer.h" void igt_global_reset_lock(struct drm_i915_private *i915) { @@ -42,3 +43,11 @@ void igt_global_reset_unlock(struct drm_i915_private *i915) clear_bit(I915_RESET_BACKOFF, &i915->gpu_error.flags); wake_up_all(&i915->gpu_error.reset_queue); } + +bool igt_force_reset(struct drm_i915_private *i915) +{ + i915_gem_set_wedged(i915); + i915_reset(i915, 0, NULL); + + return !i915_reset_failed(i915); +} diff --git a/drivers/gpu/drm/i915/selftests/igt_reset.h b/drivers/gpu/drm/i915/selftests/igt_reset.h index 5f0234d045d5..363bd853e50f 100644 --- a/drivers/gpu/drm/i915/selftests/igt_reset.h +++ b/drivers/gpu/drm/i915/selftests/igt_reset.h @@ -11,5 +11,6 @@ void igt_global_reset_lock(struct drm_i915_private *i915); void igt_global_reset_unlock(struct drm_i915_private *i915); +bool igt_force_reset(struct drm_i915_private *i915); #endif diff --git a/drivers/gpu/drm/i915/selftests/igt_spinner.c b/drivers/gpu/drm/i915/selftests/igt_spinner.c index 16890dfe74c0..1e59b543cf27 100644 --- a/drivers/gpu/drm/i915/selftests/igt_spinner.c +++ b/drivers/gpu/drm/i915/selftests/igt_spinner.c @@ -4,6 +4,8 @@ * Copyright © 2018 Intel Corporation */ +#include "gem/selftests/igt_gem_utils.h" + #include "igt_spinner.h" int igt_spinner_init(struct igt_spinner *spin, struct drm_i915_private *i915) @@ -74,16 +76,11 @@ static int move_to_active(struct i915_vma *vma, { int err; + i915_vma_lock(vma); err = i915_vma_move_to_active(vma, rq, flags); - if (err) - return err; - - if (!i915_gem_object_has_active_reference(vma->obj)) { - i915_gem_object_get(vma->obj); - i915_gem_object_set_active_reference(vma->obj); - } + i915_vma_unlock(vma); - return 0; + return err; } struct i915_request * @@ -92,17 +89,16 @@ igt_spinner_create_request(struct igt_spinner *spin, struct intel_engine_cs *engine, u32 arbitration_command) { - struct i915_address_space *vm = &ctx->ppgtt->vm; struct i915_request *rq = NULL; struct i915_vma *hws, *vma; u32 *batch; int err; - vma = i915_vma_instance(spin->obj, vm, NULL); + vma = i915_vma_instance(spin->obj, ctx->vm, NULL); if (IS_ERR(vma)) return ERR_CAST(vma); - hws = i915_vma_instance(spin->hws, vm, NULL); + hws = i915_vma_instance(spin->hws, ctx->vm, NULL); if (IS_ERR(hws)) return ERR_CAST(hws); @@ -114,7 +110,7 @@ igt_spinner_create_request(struct igt_spinner *spin, if (err) goto unpin_vma; - rq = i915_request_alloc(engine, ctx); + rq = igt_request_alloc(ctx, engine); if (IS_ERR(rq)) { err = PTR_ERR(rq); goto unpin_hws; diff --git a/drivers/gpu/drm/i915/selftests/igt_spinner.h b/drivers/gpu/drm/i915/selftests/igt_spinner.h index 391777c76dc7..34a88ac9b47a 100644 --- a/drivers/gpu/drm/i915/selftests/igt_spinner.h +++ b/drivers/gpu/drm/i915/selftests/igt_spinner.h @@ -7,12 +7,12 @@ #ifndef __I915_SELFTESTS_IGT_SPINNER_H__ #define __I915_SELFTESTS_IGT_SPINNER_H__ -#include "../i915_selftest.h" +#include "gem/i915_gem_context.h" +#include "gt/intel_engine.h" -#include "../i915_drv.h" -#include "../i915_request.h" -#include "../intel_ringbuffer.h" -#include "../i915_gem_context.h" +#include "i915_drv.h" +#include "i915_request.h" +#include "i915_selftest.h" struct igt_spinner { struct drm_i915_private *i915; diff --git a/drivers/gpu/drm/i915/selftests/intel_engine_cs.c b/drivers/gpu/drm/i915/selftests/intel_engine_cs.c deleted file mode 100644 index cfaa6b296835..000000000000 --- a/drivers/gpu/drm/i915/selftests/intel_engine_cs.c +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-License-Identifier: GPL-2.0 - * - * Copyright © 2018 Intel Corporation - */ - -#include "../i915_selftest.h" - -static int intel_mmio_bases_check(void *arg) -{ - int i, j; - - for (i = 0; i < ARRAY_SIZE(intel_engines); i++) { - const struct engine_info *info = &intel_engines[i]; - char name[INTEL_ENGINE_CS_MAX_NAME]; - u8 prev = U8_MAX; - - __sprint_engine_name(name, info); - - for (j = 0; j < MAX_MMIO_BASES; j++) { - u8 gen = info->mmio_bases[j].gen; - u32 base = info->mmio_bases[j].base; - - if (gen >= prev) { - pr_err("%s: %s: mmio base for gen %x " - "is before the one for gen %x\n", - __func__, name, prev, gen); - return -EINVAL; - } - - if (gen == 0) - break; - - if (!base) { - pr_err("%s: %s: invalid mmio base (%x) " - "for gen %x at entry %u\n", - __func__, name, base, gen, j); - return -EINVAL; - } - - prev = gen; - } - - pr_info("%s: min gen supported for %s = %d\n", - __func__, name, prev); - } - - return 0; -} - -int intel_engine_cs_mock_selftests(void) -{ - static const struct i915_subtest tests[] = { - SUBTEST(intel_mmio_bases_check), - }; - - return i915_subtests(tests, NULL); -} diff --git a/drivers/gpu/drm/i915/selftests/intel_guc.c b/drivers/gpu/drm/i915/selftests/intel_guc.c index b05a21eaa8f4..6ca8584cd64c 100644 --- a/drivers/gpu/drm/i915/selftests/intel_guc.c +++ b/drivers/gpu/drm/i915/selftests/intel_guc.c @@ -22,7 +22,8 @@ * */ -#include "../i915_selftest.h" +#include "i915_selftest.h" +#include "gem/i915_gem_pm.h" /* max doorbell number + negative test for each client type */ #define ATTEMPTS (GUC_NUM_DOORBELLS + GUC_CLIENT_PRIORITY_NUM) @@ -143,7 +144,7 @@ static int igt_guc_clients(void *args) GEM_BUG_ON(!HAS_GUC(dev_priv)); mutex_lock(&dev_priv->drm.struct_mutex); - wakeref = intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); guc = &dev_priv->guc; if (!guc) { @@ -226,7 +227,7 @@ out: guc_clients_create(guc); guc_clients_enable(guc); unlock: - intel_runtime_pm_put(dev_priv, wakeref); + intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref); mutex_unlock(&dev_priv->drm.struct_mutex); return err; } @@ -246,7 +247,7 @@ static int igt_guc_doorbells(void *arg) GEM_BUG_ON(!HAS_GUC(dev_priv)); mutex_lock(&dev_priv->drm.struct_mutex); - wakeref = intel_runtime_pm_get(dev_priv); + wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); guc = &dev_priv->guc; if (!guc) { @@ -339,7 +340,7 @@ out: guc_client_free(clients[i]); } unlock: - intel_runtime_pm_put(dev_priv, wakeref); + intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref); mutex_unlock(&dev_priv->drm.struct_mutex); return err; } diff --git a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c deleted file mode 100644 index 050bd1e19e02..000000000000 --- a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c +++ /dev/null @@ -1,1919 +0,0 @@ -/* - * Copyright © 2016 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#include <linux/kthread.h> - -#include "../i915_selftest.h" -#include "i915_random.h" -#include "igt_flush_test.h" -#include "igt_reset.h" -#include "igt_wedge_me.h" - -#include "mock_context.h" -#include "mock_drm.h" - -#define IGT_IDLE_TIMEOUT 50 /* ms; time to wait after flushing between tests */ - -struct hang { - struct drm_i915_private *i915; - struct drm_i915_gem_object *hws; - struct drm_i915_gem_object *obj; - struct i915_gem_context *ctx; - u32 *seqno; - u32 *batch; -}; - -static int hang_init(struct hang *h, struct drm_i915_private *i915) -{ - void *vaddr; - int err; - - memset(h, 0, sizeof(*h)); - h->i915 = i915; - - h->ctx = kernel_context(i915); - if (IS_ERR(h->ctx)) - return PTR_ERR(h->ctx); - - GEM_BUG_ON(i915_gem_context_is_bannable(h->ctx)); - - h->hws = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(h->hws)) { - err = PTR_ERR(h->hws); - goto err_ctx; - } - - h->obj = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(h->obj)) { - err = PTR_ERR(h->obj); - goto err_hws; - } - - i915_gem_object_set_cache_coherency(h->hws, I915_CACHE_LLC); - vaddr = i915_gem_object_pin_map(h->hws, I915_MAP_WB); - if (IS_ERR(vaddr)) { - err = PTR_ERR(vaddr); - goto err_obj; - } - h->seqno = memset(vaddr, 0xff, PAGE_SIZE); - - vaddr = i915_gem_object_pin_map(h->obj, - i915_coherent_map_type(i915)); - if (IS_ERR(vaddr)) { - err = PTR_ERR(vaddr); - goto err_unpin_hws; - } - h->batch = vaddr; - - return 0; - -err_unpin_hws: - i915_gem_object_unpin_map(h->hws); -err_obj: - i915_gem_object_put(h->obj); -err_hws: - i915_gem_object_put(h->hws); -err_ctx: - kernel_context_close(h->ctx); - return err; -} - -static u64 hws_address(const struct i915_vma *hws, - const struct i915_request *rq) -{ - return hws->node.start + offset_in_page(sizeof(u32)*rq->fence.context); -} - -static int move_to_active(struct i915_vma *vma, - struct i915_request *rq, - unsigned int flags) -{ - int err; - - err = i915_vma_move_to_active(vma, rq, flags); - if (err) - return err; - - if (!i915_gem_object_has_active_reference(vma->obj)) { - i915_gem_object_get(vma->obj); - i915_gem_object_set_active_reference(vma->obj); - } - - return 0; -} - -static struct i915_request * -hang_create_request(struct hang *h, struct intel_engine_cs *engine) -{ - struct drm_i915_private *i915 = h->i915; - struct i915_address_space *vm = - h->ctx->ppgtt ? &h->ctx->ppgtt->vm : &i915->ggtt.vm; - struct i915_request *rq = NULL; - struct i915_vma *hws, *vma; - unsigned int flags; - u32 *batch; - int err; - - if (i915_gem_object_is_active(h->obj)) { - struct drm_i915_gem_object *obj; - void *vaddr; - - obj = i915_gem_object_create_internal(h->i915, PAGE_SIZE); - if (IS_ERR(obj)) - return ERR_CAST(obj); - - vaddr = i915_gem_object_pin_map(obj, - i915_coherent_map_type(h->i915)); - if (IS_ERR(vaddr)) { - i915_gem_object_put(obj); - return ERR_CAST(vaddr); - } - - i915_gem_object_unpin_map(h->obj); - i915_gem_object_put(h->obj); - - h->obj = obj; - h->batch = vaddr; - } - - vma = i915_vma_instance(h->obj, vm, NULL); - if (IS_ERR(vma)) - return ERR_CAST(vma); - - hws = i915_vma_instance(h->hws, vm, NULL); - if (IS_ERR(hws)) - return ERR_CAST(hws); - - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) - return ERR_PTR(err); - - err = i915_vma_pin(hws, 0, 0, PIN_USER); - if (err) - goto unpin_vma; - - rq = i915_request_alloc(engine, h->ctx); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto unpin_hws; - } - - err = move_to_active(vma, rq, 0); - if (err) - goto cancel_rq; - - err = move_to_active(hws, rq, 0); - if (err) - goto cancel_rq; - - batch = h->batch; - if (INTEL_GEN(i915) >= 8) { - *batch++ = MI_STORE_DWORD_IMM_GEN4; - *batch++ = lower_32_bits(hws_address(hws, rq)); - *batch++ = upper_32_bits(hws_address(hws, rq)); - *batch++ = rq->fence.seqno; - *batch++ = MI_ARB_CHECK; - - memset(batch, 0, 1024); - batch += 1024 / sizeof(*batch); - - *batch++ = MI_ARB_CHECK; - *batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1; - *batch++ = lower_32_bits(vma->node.start); - *batch++ = upper_32_bits(vma->node.start); - } else if (INTEL_GEN(i915) >= 6) { - *batch++ = MI_STORE_DWORD_IMM_GEN4; - *batch++ = 0; - *batch++ = lower_32_bits(hws_address(hws, rq)); - *batch++ = rq->fence.seqno; - *batch++ = MI_ARB_CHECK; - - memset(batch, 0, 1024); - batch += 1024 / sizeof(*batch); - - *batch++ = MI_ARB_CHECK; - *batch++ = MI_BATCH_BUFFER_START | 1 << 8; - *batch++ = lower_32_bits(vma->node.start); - } else if (INTEL_GEN(i915) >= 4) { - *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; - *batch++ = 0; - *batch++ = lower_32_bits(hws_address(hws, rq)); - *batch++ = rq->fence.seqno; - *batch++ = MI_ARB_CHECK; - - memset(batch, 0, 1024); - batch += 1024 / sizeof(*batch); - - *batch++ = MI_ARB_CHECK; - *batch++ = MI_BATCH_BUFFER_START | 2 << 6; - *batch++ = lower_32_bits(vma->node.start); - } else { - *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; - *batch++ = lower_32_bits(hws_address(hws, rq)); - *batch++ = rq->fence.seqno; - *batch++ = MI_ARB_CHECK; - - memset(batch, 0, 1024); - batch += 1024 / sizeof(*batch); - - *batch++ = MI_ARB_CHECK; - *batch++ = MI_BATCH_BUFFER_START | 2 << 6; - *batch++ = lower_32_bits(vma->node.start); - } - *batch++ = MI_BATCH_BUFFER_END; /* not reached */ - i915_gem_chipset_flush(h->i915); - - if (rq->engine->emit_init_breadcrumb) { - err = rq->engine->emit_init_breadcrumb(rq); - if (err) - goto cancel_rq; - } - - flags = 0; - if (INTEL_GEN(vm->i915) <= 5) - flags |= I915_DISPATCH_SECURE; - - err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags); - -cancel_rq: - if (err) { - i915_request_skip(rq, err); - i915_request_add(rq); - } -unpin_hws: - i915_vma_unpin(hws); -unpin_vma: - i915_vma_unpin(vma); - return err ? ERR_PTR(err) : rq; -} - -static u32 hws_seqno(const struct hang *h, const struct i915_request *rq) -{ - return READ_ONCE(h->seqno[rq->fence.context % (PAGE_SIZE/sizeof(u32))]); -} - -static void hang_fini(struct hang *h) -{ - *h->batch = MI_BATCH_BUFFER_END; - i915_gem_chipset_flush(h->i915); - - i915_gem_object_unpin_map(h->obj); - i915_gem_object_put(h->obj); - - i915_gem_object_unpin_map(h->hws); - i915_gem_object_put(h->hws); - - kernel_context_close(h->ctx); - - igt_flush_test(h->i915, I915_WAIT_LOCKED); -} - -static bool wait_until_running(struct hang *h, struct i915_request *rq) -{ - return !(wait_for_us(i915_seqno_passed(hws_seqno(h, rq), - rq->fence.seqno), - 10) && - wait_for(i915_seqno_passed(hws_seqno(h, rq), - rq->fence.seqno), - 1000)); -} - -static int igt_hang_sanitycheck(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct i915_request *rq; - struct intel_engine_cs *engine; - enum intel_engine_id id; - struct hang h; - int err; - - /* Basic check that we can execute our hanging batch */ - - mutex_lock(&i915->drm.struct_mutex); - err = hang_init(&h, i915); - if (err) - goto unlock; - - for_each_engine(engine, i915, id) { - struct igt_wedge_me w; - long timeout; - - if (!intel_engine_can_store_dword(engine)) - continue; - - rq = hang_create_request(&h, engine); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - pr_err("Failed to create request for %s, err=%d\n", - engine->name, err); - goto fini; - } - - i915_request_get(rq); - - *h.batch = MI_BATCH_BUFFER_END; - i915_gem_chipset_flush(i915); - - i915_request_add(rq); - - timeout = 0; - igt_wedge_on_timeout(&w, i915, HZ / 10 /* 100ms timeout*/) - timeout = i915_request_wait(rq, - I915_WAIT_LOCKED, - MAX_SCHEDULE_TIMEOUT); - if (i915_reset_failed(i915)) - timeout = -EIO; - - i915_request_put(rq); - - if (timeout < 0) { - err = timeout; - pr_err("Wait for request failed on %s, err=%d\n", - engine->name, err); - goto fini; - } - } - -fini: - hang_fini(&h); -unlock: - mutex_unlock(&i915->drm.struct_mutex); - return err; -} - -static int igt_global_reset(void *arg) -{ - struct drm_i915_private *i915 = arg; - unsigned int reset_count; - int err = 0; - - /* Check that we can issue a global GPU reset */ - - igt_global_reset_lock(i915); - - reset_count = i915_reset_count(&i915->gpu_error); - - i915_reset(i915, ALL_ENGINES, NULL); - - if (i915_reset_count(&i915->gpu_error) == reset_count) { - pr_err("No GPU reset recorded!\n"); - err = -EINVAL; - } - - igt_global_reset_unlock(i915); - - if (i915_reset_failed(i915)) - err = -EIO; - - return err; -} - -static int igt_wedged_reset(void *arg) -{ - struct drm_i915_private *i915 = arg; - intel_wakeref_t wakeref; - - /* Check that we can recover a wedged device with a GPU reset */ - - igt_global_reset_lock(i915); - wakeref = intel_runtime_pm_get(i915); - - i915_gem_set_wedged(i915); - - GEM_BUG_ON(!i915_reset_failed(i915)); - i915_reset(i915, ALL_ENGINES, NULL); - - intel_runtime_pm_put(i915, wakeref); - igt_global_reset_unlock(i915); - - return i915_reset_failed(i915) ? -EIO : 0; -} - -static bool wait_for_idle(struct intel_engine_cs *engine) -{ - return wait_for(intel_engine_is_idle(engine), IGT_IDLE_TIMEOUT) == 0; -} - -static int igt_reset_nop(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *engine; - struct i915_gem_context *ctx; - unsigned int reset_count, count; - enum intel_engine_id id; - intel_wakeref_t wakeref; - struct drm_file *file; - IGT_TIMEOUT(end_time); - int err = 0; - - /* Check that we can reset during non-user portions of requests */ - - file = mock_file(i915); - if (IS_ERR(file)) - return PTR_ERR(file); - - mutex_lock(&i915->drm.struct_mutex); - ctx = live_context(i915, file); - mutex_unlock(&i915->drm.struct_mutex); - if (IS_ERR(ctx)) { - err = PTR_ERR(ctx); - goto out; - } - - i915_gem_context_clear_bannable(ctx); - wakeref = intel_runtime_pm_get(i915); - reset_count = i915_reset_count(&i915->gpu_error); - count = 0; - do { - mutex_lock(&i915->drm.struct_mutex); - for_each_engine(engine, i915, id) { - int i; - - for (i = 0; i < 16; i++) { - struct i915_request *rq; - - rq = i915_request_alloc(engine, ctx); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - break; - } - - i915_request_add(rq); - } - } - mutex_unlock(&i915->drm.struct_mutex); - - igt_global_reset_lock(i915); - i915_reset(i915, ALL_ENGINES, NULL); - igt_global_reset_unlock(i915); - if (i915_reset_failed(i915)) { - err = -EIO; - break; - } - - if (i915_reset_count(&i915->gpu_error) != - reset_count + ++count) { - pr_err("Full GPU reset not recorded!\n"); - err = -EINVAL; - break; - } - - if (!i915_reset_flush(i915)) { - struct drm_printer p = - drm_info_printer(i915->drm.dev); - - pr_err("%s failed to idle after reset\n", - engine->name); - intel_engine_dump(engine, &p, - "%s\n", engine->name); - - err = -EIO; - break; - } - - err = igt_flush_test(i915, 0); - if (err) - break; - } while (time_before(jiffies, end_time)); - pr_info("%s: %d resets\n", __func__, count); - - mutex_lock(&i915->drm.struct_mutex); - err = igt_flush_test(i915, I915_WAIT_LOCKED); - mutex_unlock(&i915->drm.struct_mutex); - - intel_runtime_pm_put(i915, wakeref); - -out: - mock_file_free(i915, file); - if (i915_reset_failed(i915)) - err = -EIO; - return err; -} - -static int igt_reset_nop_engine(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *engine; - struct i915_gem_context *ctx; - enum intel_engine_id id; - intel_wakeref_t wakeref; - struct drm_file *file; - int err = 0; - - /* Check that we can engine-reset during non-user portions */ - - if (!intel_has_reset_engine(i915)) - return 0; - - file = mock_file(i915); - if (IS_ERR(file)) - return PTR_ERR(file); - - mutex_lock(&i915->drm.struct_mutex); - ctx = live_context(i915, file); - mutex_unlock(&i915->drm.struct_mutex); - if (IS_ERR(ctx)) { - err = PTR_ERR(ctx); - goto out; - } - - i915_gem_context_clear_bannable(ctx); - wakeref = intel_runtime_pm_get(i915); - for_each_engine(engine, i915, id) { - unsigned int reset_count, reset_engine_count; - unsigned int count; - IGT_TIMEOUT(end_time); - - reset_count = i915_reset_count(&i915->gpu_error); - reset_engine_count = i915_reset_engine_count(&i915->gpu_error, - engine); - count = 0; - - set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); - do { - int i; - - if (!wait_for_idle(engine)) { - pr_err("%s failed to idle before reset\n", - engine->name); - err = -EIO; - break; - } - - mutex_lock(&i915->drm.struct_mutex); - for (i = 0; i < 16; i++) { - struct i915_request *rq; - - rq = i915_request_alloc(engine, ctx); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - break; - } - - i915_request_add(rq); - } - mutex_unlock(&i915->drm.struct_mutex); - - err = i915_reset_engine(engine, NULL); - if (err) { - pr_err("i915_reset_engine failed\n"); - break; - } - - if (i915_reset_count(&i915->gpu_error) != reset_count) { - pr_err("Full GPU reset recorded! (engine reset expected)\n"); - err = -EINVAL; - break; - } - - if (i915_reset_engine_count(&i915->gpu_error, engine) != - reset_engine_count + ++count) { - pr_err("%s engine reset not recorded!\n", - engine->name); - err = -EINVAL; - break; - } - - if (!i915_reset_flush(i915)) { - struct drm_printer p = - drm_info_printer(i915->drm.dev); - - pr_err("%s failed to idle after reset\n", - engine->name); - intel_engine_dump(engine, &p, - "%s\n", engine->name); - - err = -EIO; - break; - } - } while (time_before(jiffies, end_time)); - clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); - pr_info("%s(%s): %d resets\n", __func__, engine->name, count); - - if (err) - break; - - err = igt_flush_test(i915, 0); - if (err) - break; - } - - mutex_lock(&i915->drm.struct_mutex); - err = igt_flush_test(i915, I915_WAIT_LOCKED); - mutex_unlock(&i915->drm.struct_mutex); - - intel_runtime_pm_put(i915, wakeref); -out: - mock_file_free(i915, file); - if (i915_reset_failed(i915)) - err = -EIO; - return err; -} - -static int __igt_reset_engine(struct drm_i915_private *i915, bool active) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - struct hang h; - int err = 0; - - /* Check that we can issue an engine reset on an idle engine (no-op) */ - - if (!intel_has_reset_engine(i915)) - return 0; - - if (active) { - mutex_lock(&i915->drm.struct_mutex); - err = hang_init(&h, i915); - mutex_unlock(&i915->drm.struct_mutex); - if (err) - return err; - } - - for_each_engine(engine, i915, id) { - unsigned int reset_count, reset_engine_count; - IGT_TIMEOUT(end_time); - - if (active && !intel_engine_can_store_dword(engine)) - continue; - - if (!wait_for_idle(engine)) { - pr_err("%s failed to idle before reset\n", - engine->name); - err = -EIO; - break; - } - - reset_count = i915_reset_count(&i915->gpu_error); - reset_engine_count = i915_reset_engine_count(&i915->gpu_error, - engine); - - set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); - do { - if (active) { - struct i915_request *rq; - - mutex_lock(&i915->drm.struct_mutex); - rq = hang_create_request(&h, engine); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - mutex_unlock(&i915->drm.struct_mutex); - break; - } - - i915_request_get(rq); - i915_request_add(rq); - mutex_unlock(&i915->drm.struct_mutex); - - if (!wait_until_running(&h, rq)) { - struct drm_printer p = drm_info_printer(i915->drm.dev); - - pr_err("%s: Failed to start request %llx, at %x\n", - __func__, rq->fence.seqno, hws_seqno(&h, rq)); - intel_engine_dump(engine, &p, - "%s\n", engine->name); - - i915_request_put(rq); - err = -EIO; - break; - } - - i915_request_put(rq); - } - - err = i915_reset_engine(engine, NULL); - if (err) { - pr_err("i915_reset_engine failed\n"); - break; - } - - if (i915_reset_count(&i915->gpu_error) != reset_count) { - pr_err("Full GPU reset recorded! (engine reset expected)\n"); - err = -EINVAL; - break; - } - - if (i915_reset_engine_count(&i915->gpu_error, engine) != - ++reset_engine_count) { - pr_err("%s engine reset not recorded!\n", - engine->name); - err = -EINVAL; - break; - } - - if (!i915_reset_flush(i915)) { - struct drm_printer p = - drm_info_printer(i915->drm.dev); - - pr_err("%s failed to idle after reset\n", - engine->name); - intel_engine_dump(engine, &p, - "%s\n", engine->name); - - err = -EIO; - break; - } - } while (time_before(jiffies, end_time)); - clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); - - if (err) - break; - - err = igt_flush_test(i915, 0); - if (err) - break; - } - - if (i915_reset_failed(i915)) - err = -EIO; - - if (active) { - mutex_lock(&i915->drm.struct_mutex); - hang_fini(&h); - mutex_unlock(&i915->drm.struct_mutex); - } - - return err; -} - -static int igt_reset_idle_engine(void *arg) -{ - return __igt_reset_engine(arg, false); -} - -static int igt_reset_active_engine(void *arg) -{ - return __igt_reset_engine(arg, true); -} - -struct active_engine { - struct task_struct *task; - struct intel_engine_cs *engine; - unsigned long resets; - unsigned int flags; -}; - -#define TEST_ACTIVE BIT(0) -#define TEST_OTHERS BIT(1) -#define TEST_SELF BIT(2) -#define TEST_PRIORITY BIT(3) - -static int active_request_put(struct i915_request *rq) -{ - int err = 0; - - if (!rq) - return 0; - - if (i915_request_wait(rq, 0, 5 * HZ) < 0) { - GEM_TRACE("%s timed out waiting for completion of fence %llx:%lld\n", - rq->engine->name, - rq->fence.context, - rq->fence.seqno); - GEM_TRACE_DUMP(); - - i915_gem_set_wedged(rq->i915); - err = -EIO; - } - - i915_request_put(rq); - - return err; -} - -static int active_engine(void *data) -{ - I915_RND_STATE(prng); - struct active_engine *arg = data; - struct intel_engine_cs *engine = arg->engine; - struct i915_request *rq[8] = {}; - struct i915_gem_context *ctx[ARRAY_SIZE(rq)]; - struct drm_file *file; - unsigned long count = 0; - int err = 0; - - file = mock_file(engine->i915); - if (IS_ERR(file)) - return PTR_ERR(file); - - for (count = 0; count < ARRAY_SIZE(ctx); count++) { - mutex_lock(&engine->i915->drm.struct_mutex); - ctx[count] = live_context(engine->i915, file); - mutex_unlock(&engine->i915->drm.struct_mutex); - if (IS_ERR(ctx[count])) { - err = PTR_ERR(ctx[count]); - while (--count) - i915_gem_context_put(ctx[count]); - goto err_file; - } - } - - while (!kthread_should_stop()) { - unsigned int idx = count++ & (ARRAY_SIZE(rq) - 1); - struct i915_request *old = rq[idx]; - struct i915_request *new; - - mutex_lock(&engine->i915->drm.struct_mutex); - new = i915_request_alloc(engine, ctx[idx]); - if (IS_ERR(new)) { - mutex_unlock(&engine->i915->drm.struct_mutex); - err = PTR_ERR(new); - break; - } - - if (arg->flags & TEST_PRIORITY) - ctx[idx]->sched.priority = - i915_prandom_u32_max_state(512, &prng); - - rq[idx] = i915_request_get(new); - i915_request_add(new); - mutex_unlock(&engine->i915->drm.struct_mutex); - - err = active_request_put(old); - if (err) - break; - - cond_resched(); - } - - for (count = 0; count < ARRAY_SIZE(rq); count++) { - int err__ = active_request_put(rq[count]); - - /* Keep the first error */ - if (!err) - err = err__; - } - -err_file: - mock_file_free(engine->i915, file); - return err; -} - -static int __igt_reset_engines(struct drm_i915_private *i915, - const char *test_name, - unsigned int flags) -{ - struct intel_engine_cs *engine, *other; - enum intel_engine_id id, tmp; - struct hang h; - int err = 0; - - /* Check that issuing a reset on one engine does not interfere - * with any other engine. - */ - - if (!intel_has_reset_engine(i915)) - return 0; - - if (flags & TEST_ACTIVE) { - mutex_lock(&i915->drm.struct_mutex); - err = hang_init(&h, i915); - mutex_unlock(&i915->drm.struct_mutex); - if (err) - return err; - - if (flags & TEST_PRIORITY) - h.ctx->sched.priority = 1024; - } - - for_each_engine(engine, i915, id) { - struct active_engine threads[I915_NUM_ENGINES] = {}; - unsigned long global = i915_reset_count(&i915->gpu_error); - unsigned long count = 0, reported; - IGT_TIMEOUT(end_time); - - if (flags & TEST_ACTIVE && - !intel_engine_can_store_dword(engine)) - continue; - - if (!wait_for_idle(engine)) { - pr_err("i915_reset_engine(%s:%s): failed to idle before reset\n", - engine->name, test_name); - err = -EIO; - break; - } - - memset(threads, 0, sizeof(threads)); - for_each_engine(other, i915, tmp) { - struct task_struct *tsk; - - threads[tmp].resets = - i915_reset_engine_count(&i915->gpu_error, - other); - - if (!(flags & TEST_OTHERS)) - continue; - - if (other == engine && !(flags & TEST_SELF)) - continue; - - threads[tmp].engine = other; - threads[tmp].flags = flags; - - tsk = kthread_run(active_engine, &threads[tmp], - "igt/%s", other->name); - if (IS_ERR(tsk)) { - err = PTR_ERR(tsk); - goto unwind; - } - - threads[tmp].task = tsk; - get_task_struct(tsk); - } - - set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); - do { - struct i915_request *rq = NULL; - - if (flags & TEST_ACTIVE) { - mutex_lock(&i915->drm.struct_mutex); - rq = hang_create_request(&h, engine); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - mutex_unlock(&i915->drm.struct_mutex); - break; - } - - i915_request_get(rq); - i915_request_add(rq); - mutex_unlock(&i915->drm.struct_mutex); - - if (!wait_until_running(&h, rq)) { - struct drm_printer p = drm_info_printer(i915->drm.dev); - - pr_err("%s: Failed to start request %llx, at %x\n", - __func__, rq->fence.seqno, hws_seqno(&h, rq)); - intel_engine_dump(engine, &p, - "%s\n", engine->name); - - i915_request_put(rq); - err = -EIO; - break; - } - } - - err = i915_reset_engine(engine, NULL); - if (err) { - pr_err("i915_reset_engine(%s:%s): failed, err=%d\n", - engine->name, test_name, err); - break; - } - - count++; - - if (rq) { - if (i915_request_wait(rq, 0, HZ / 5) < 0) { - struct drm_printer p = - drm_info_printer(i915->drm.dev); - - pr_err("i915_reset_engine(%s:%s):" - " failed to complete request after reset\n", - engine->name, test_name); - intel_engine_dump(engine, &p, - "%s\n", engine->name); - i915_request_put(rq); - - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - err = -EIO; - break; - } - - i915_request_put(rq); - } - - if (!(flags & TEST_SELF) && !wait_for_idle(engine)) { - struct drm_printer p = - drm_info_printer(i915->drm.dev); - - pr_err("i915_reset_engine(%s:%s):" - " failed to idle after reset\n", - engine->name, test_name); - intel_engine_dump(engine, &p, - "%s\n", engine->name); - - err = -EIO; - break; - } - } while (time_before(jiffies, end_time)); - clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); - pr_info("i915_reset_engine(%s:%s): %lu resets\n", - engine->name, test_name, count); - - reported = i915_reset_engine_count(&i915->gpu_error, engine); - reported -= threads[engine->id].resets; - if (reported != count) { - pr_err("i915_reset_engine(%s:%s): reset %lu times, but reported %lu\n", - engine->name, test_name, count, reported); - if (!err) - err = -EINVAL; - } - -unwind: - for_each_engine(other, i915, tmp) { - int ret; - - if (!threads[tmp].task) - continue; - - ret = kthread_stop(threads[tmp].task); - if (ret) { - pr_err("kthread for other engine %s failed, err=%d\n", - other->name, ret); - if (!err) - err = ret; - } - put_task_struct(threads[tmp].task); - - if (other != engine && - threads[tmp].resets != - i915_reset_engine_count(&i915->gpu_error, other)) { - pr_err("Innocent engine %s was reset (count=%ld)\n", - other->name, - i915_reset_engine_count(&i915->gpu_error, - other) - - threads[tmp].resets); - if (!err) - err = -EINVAL; - } - } - - if (global != i915_reset_count(&i915->gpu_error)) { - pr_err("Global reset (count=%ld)!\n", - i915_reset_count(&i915->gpu_error) - global); - if (!err) - err = -EINVAL; - } - - if (err) - break; - - err = igt_flush_test(i915, 0); - if (err) - break; - } - - if (i915_reset_failed(i915)) - err = -EIO; - - if (flags & TEST_ACTIVE) { - mutex_lock(&i915->drm.struct_mutex); - hang_fini(&h); - mutex_unlock(&i915->drm.struct_mutex); - } - - return err; -} - -static int igt_reset_engines(void *arg) -{ - static const struct { - const char *name; - unsigned int flags; - } phases[] = { - { "idle", 0 }, - { "active", TEST_ACTIVE }, - { "others-idle", TEST_OTHERS }, - { "others-active", TEST_OTHERS | TEST_ACTIVE }, - { - "others-priority", - TEST_OTHERS | TEST_ACTIVE | TEST_PRIORITY - }, - { - "self-priority", - TEST_OTHERS | TEST_ACTIVE | TEST_PRIORITY | TEST_SELF, - }, - { } - }; - struct drm_i915_private *i915 = arg; - typeof(*phases) *p; - int err; - - for (p = phases; p->name; p++) { - if (p->flags & TEST_PRIORITY) { - if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY)) - continue; - } - - err = __igt_reset_engines(arg, p->name, p->flags); - if (err) - return err; - } - - return 0; -} - -static u32 fake_hangcheck(struct drm_i915_private *i915, - intel_engine_mask_t mask) -{ - u32 count = i915_reset_count(&i915->gpu_error); - - i915_reset(i915, mask, NULL); - - return count; -} - -static int igt_reset_wait(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct i915_request *rq; - unsigned int reset_count; - struct hang h; - long timeout; - int err; - - if (!intel_engine_can_store_dword(i915->engine[RCS0])) - return 0; - - /* Check that we detect a stuck waiter and issue a reset */ - - igt_global_reset_lock(i915); - - mutex_lock(&i915->drm.struct_mutex); - err = hang_init(&h, i915); - if (err) - goto unlock; - - rq = hang_create_request(&h, i915->engine[RCS0]); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto fini; - } - - i915_request_get(rq); - i915_request_add(rq); - - if (!wait_until_running(&h, rq)) { - struct drm_printer p = drm_info_printer(i915->drm.dev); - - pr_err("%s: Failed to start request %llx, at %x\n", - __func__, rq->fence.seqno, hws_seqno(&h, rq)); - intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name); - - i915_gem_set_wedged(i915); - - err = -EIO; - goto out_rq; - } - - reset_count = fake_hangcheck(i915, ALL_ENGINES); - - timeout = i915_request_wait(rq, I915_WAIT_LOCKED, 10); - if (timeout < 0) { - pr_err("i915_request_wait failed on a stuck request: err=%ld\n", - timeout); - err = timeout; - goto out_rq; - } - - if (i915_reset_count(&i915->gpu_error) == reset_count) { - pr_err("No GPU reset recorded!\n"); - err = -EINVAL; - goto out_rq; - } - -out_rq: - i915_request_put(rq); -fini: - hang_fini(&h); -unlock: - mutex_unlock(&i915->drm.struct_mutex); - igt_global_reset_unlock(i915); - - if (i915_reset_failed(i915)) - return -EIO; - - return err; -} - -struct evict_vma { - struct completion completion; - struct i915_vma *vma; -}; - -static int evict_vma(void *data) -{ - struct evict_vma *arg = data; - struct i915_address_space *vm = arg->vma->vm; - struct drm_i915_private *i915 = vm->i915; - struct drm_mm_node evict = arg->vma->node; - int err; - - complete(&arg->completion); - - mutex_lock(&i915->drm.struct_mutex); - err = i915_gem_evict_for_node(vm, &evict, 0); - mutex_unlock(&i915->drm.struct_mutex); - - return err; -} - -static int evict_fence(void *data) -{ - struct evict_vma *arg = data; - struct drm_i915_private *i915 = arg->vma->vm->i915; - int err; - - complete(&arg->completion); - - mutex_lock(&i915->drm.struct_mutex); - - /* Mark the fence register as dirty to force the mmio update. */ - err = i915_gem_object_set_tiling(arg->vma->obj, I915_TILING_Y, 512); - if (err) { - pr_err("Invalid Y-tiling settings; err:%d\n", err); - goto out_unlock; - } - - err = i915_vma_pin_fence(arg->vma); - if (err) { - pr_err("Unable to pin Y-tiled fence; err:%d\n", err); - goto out_unlock; - } - - i915_vma_unpin_fence(arg->vma); - -out_unlock: - mutex_unlock(&i915->drm.struct_mutex); - - return err; -} - -static int __igt_reset_evict_vma(struct drm_i915_private *i915, - struct i915_address_space *vm, - int (*fn)(void *), - unsigned int flags) -{ - struct drm_i915_gem_object *obj; - struct task_struct *tsk = NULL; - struct i915_request *rq; - struct evict_vma arg; - struct hang h; - int err; - - if (!intel_engine_can_store_dword(i915->engine[RCS0])) - return 0; - - /* Check that we can recover an unbind stuck on a hanging request */ - - mutex_lock(&i915->drm.struct_mutex); - err = hang_init(&h, i915); - if (err) - goto unlock; - - obj = i915_gem_object_create_internal(i915, SZ_1M); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto fini; - } - - if (flags & EXEC_OBJECT_NEEDS_FENCE) { - err = i915_gem_object_set_tiling(obj, I915_TILING_X, 512); - if (err) { - pr_err("Invalid X-tiling settings; err:%d\n", err); - goto out_obj; - } - } - - arg.vma = i915_vma_instance(obj, vm, NULL); - if (IS_ERR(arg.vma)) { - err = PTR_ERR(arg.vma); - goto out_obj; - } - - rq = hang_create_request(&h, i915->engine[RCS0]); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto out_obj; - } - - err = i915_vma_pin(arg.vma, 0, 0, - i915_vma_is_ggtt(arg.vma) ? - PIN_GLOBAL | PIN_MAPPABLE : - PIN_USER); - if (err) { - i915_request_add(rq); - goto out_obj; - } - - if (flags & EXEC_OBJECT_NEEDS_FENCE) { - err = i915_vma_pin_fence(arg.vma); - if (err) { - pr_err("Unable to pin X-tiled fence; err:%d\n", err); - i915_vma_unpin(arg.vma); - i915_request_add(rq); - goto out_obj; - } - } - - err = i915_vma_move_to_active(arg.vma, rq, flags); - - if (flags & EXEC_OBJECT_NEEDS_FENCE) - i915_vma_unpin_fence(arg.vma); - i915_vma_unpin(arg.vma); - - i915_request_get(rq); - i915_request_add(rq); - if (err) - goto out_rq; - - mutex_unlock(&i915->drm.struct_mutex); - - if (!wait_until_running(&h, rq)) { - struct drm_printer p = drm_info_printer(i915->drm.dev); - - pr_err("%s: Failed to start request %llx, at %x\n", - __func__, rq->fence.seqno, hws_seqno(&h, rq)); - intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name); - - i915_gem_set_wedged(i915); - goto out_reset; - } - - init_completion(&arg.completion); - - tsk = kthread_run(fn, &arg, "igt/evict_vma"); - if (IS_ERR(tsk)) { - err = PTR_ERR(tsk); - tsk = NULL; - goto out_reset; - } - get_task_struct(tsk); - - wait_for_completion(&arg.completion); - - if (wait_for(!list_empty(&rq->fence.cb_list), 10)) { - struct drm_printer p = drm_info_printer(i915->drm.dev); - - pr_err("igt/evict_vma kthread did not wait\n"); - intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name); - - i915_gem_set_wedged(i915); - goto out_reset; - } - -out_reset: - igt_global_reset_lock(i915); - fake_hangcheck(rq->i915, rq->engine->mask); - igt_global_reset_unlock(i915); - - if (tsk) { - struct igt_wedge_me w; - - /* The reset, even indirectly, should take less than 10ms. */ - igt_wedge_on_timeout(&w, i915, HZ / 10 /* 100ms timeout*/) - err = kthread_stop(tsk); - - put_task_struct(tsk); - } - - mutex_lock(&i915->drm.struct_mutex); -out_rq: - i915_request_put(rq); -out_obj: - i915_gem_object_put(obj); -fini: - hang_fini(&h); -unlock: - mutex_unlock(&i915->drm.struct_mutex); - - if (i915_reset_failed(i915)) - return -EIO; - - return err; -} - -static int igt_reset_evict_ggtt(void *arg) -{ - struct drm_i915_private *i915 = arg; - - return __igt_reset_evict_vma(i915, &i915->ggtt.vm, - evict_vma, EXEC_OBJECT_WRITE); -} - -static int igt_reset_evict_ppgtt(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct i915_gem_context *ctx; - struct drm_file *file; - int err; - - file = mock_file(i915); - if (IS_ERR(file)) - return PTR_ERR(file); - - mutex_lock(&i915->drm.struct_mutex); - ctx = live_context(i915, file); - mutex_unlock(&i915->drm.struct_mutex); - if (IS_ERR(ctx)) { - err = PTR_ERR(ctx); - goto out; - } - - err = 0; - if (ctx->ppgtt) /* aliasing == global gtt locking, covered above */ - err = __igt_reset_evict_vma(i915, &ctx->ppgtt->vm, - evict_vma, EXEC_OBJECT_WRITE); - -out: - mock_file_free(i915, file); - return err; -} - -static int igt_reset_evict_fence(void *arg) -{ - struct drm_i915_private *i915 = arg; - - return __igt_reset_evict_vma(i915, &i915->ggtt.vm, - evict_fence, EXEC_OBJECT_NEEDS_FENCE); -} - -static int wait_for_others(struct drm_i915_private *i915, - struct intel_engine_cs *exclude) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - for_each_engine(engine, i915, id) { - if (engine == exclude) - continue; - - if (!wait_for_idle(engine)) - return -EIO; - } - - return 0; -} - -static int igt_reset_queue(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *engine; - enum intel_engine_id id; - struct hang h; - int err; - - /* Check that we replay pending requests following a hang */ - - igt_global_reset_lock(i915); - - mutex_lock(&i915->drm.struct_mutex); - err = hang_init(&h, i915); - if (err) - goto unlock; - - for_each_engine(engine, i915, id) { - struct i915_request *prev; - IGT_TIMEOUT(end_time); - unsigned int count; - - if (!intel_engine_can_store_dword(engine)) - continue; - - prev = hang_create_request(&h, engine); - if (IS_ERR(prev)) { - err = PTR_ERR(prev); - goto fini; - } - - i915_request_get(prev); - i915_request_add(prev); - - count = 0; - do { - struct i915_request *rq; - unsigned int reset_count; - - rq = hang_create_request(&h, engine); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto fini; - } - - i915_request_get(rq); - i915_request_add(rq); - - /* - * XXX We don't handle resetting the kernel context - * very well. If we trigger a device reset twice in - * quick succession while the kernel context is - * executing, we may end up skipping the breadcrumb. - * This is really only a problem for the selftest as - * normally there is a large interlude between resets - * (hangcheck), or we focus on resetting just one - * engine and so avoid repeatedly resetting innocents. - */ - err = wait_for_others(i915, engine); - if (err) { - pr_err("%s(%s): Failed to idle other inactive engines after device reset\n", - __func__, engine->name); - i915_request_put(rq); - i915_request_put(prev); - - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - goto fini; - } - - if (!wait_until_running(&h, prev)) { - struct drm_printer p = drm_info_printer(i915->drm.dev); - - pr_err("%s(%s): Failed to start request %llx, at %x\n", - __func__, engine->name, - prev->fence.seqno, hws_seqno(&h, prev)); - intel_engine_dump(engine, &p, - "%s\n", engine->name); - - i915_request_put(rq); - i915_request_put(prev); - - i915_gem_set_wedged(i915); - - err = -EIO; - goto fini; - } - - reset_count = fake_hangcheck(i915, BIT(id)); - - if (prev->fence.error != -EIO) { - pr_err("GPU reset not recorded on hanging request [fence.error=%d]!\n", - prev->fence.error); - i915_request_put(rq); - i915_request_put(prev); - err = -EINVAL; - goto fini; - } - - if (rq->fence.error) { - pr_err("Fence error status not zero [%d] after unrelated reset\n", - rq->fence.error); - i915_request_put(rq); - i915_request_put(prev); - err = -EINVAL; - goto fini; - } - - if (i915_reset_count(&i915->gpu_error) == reset_count) { - pr_err("No GPU reset recorded!\n"); - i915_request_put(rq); - i915_request_put(prev); - err = -EINVAL; - goto fini; - } - - i915_request_put(prev); - prev = rq; - count++; - } while (time_before(jiffies, end_time)); - pr_info("%s: Completed %d resets\n", engine->name, count); - - *h.batch = MI_BATCH_BUFFER_END; - i915_gem_chipset_flush(i915); - - i915_request_put(prev); - - err = igt_flush_test(i915, I915_WAIT_LOCKED); - if (err) - break; - } - -fini: - hang_fini(&h); -unlock: - mutex_unlock(&i915->drm.struct_mutex); - igt_global_reset_unlock(i915); - - if (i915_reset_failed(i915)) - return -EIO; - - return err; -} - -static int igt_handle_error(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *engine = i915->engine[RCS0]; - struct hang h; - struct i915_request *rq; - struct i915_gpu_state *error; - int err; - - /* Check that we can issue a global GPU and engine reset */ - - if (!intel_has_reset_engine(i915)) - return 0; - - if (!engine || !intel_engine_can_store_dword(engine)) - return 0; - - mutex_lock(&i915->drm.struct_mutex); - - err = hang_init(&h, i915); - if (err) - goto err_unlock; - - rq = hang_create_request(&h, engine); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_fini; - } - - i915_request_get(rq); - i915_request_add(rq); - - if (!wait_until_running(&h, rq)) { - struct drm_printer p = drm_info_printer(i915->drm.dev); - - pr_err("%s: Failed to start request %llx, at %x\n", - __func__, rq->fence.seqno, hws_seqno(&h, rq)); - intel_engine_dump(rq->engine, &p, "%s\n", rq->engine->name); - - i915_gem_set_wedged(i915); - - err = -EIO; - goto err_request; - } - - mutex_unlock(&i915->drm.struct_mutex); - - /* Temporarily disable error capture */ - error = xchg(&i915->gpu_error.first_error, (void *)-1); - - i915_handle_error(i915, engine->mask, 0, NULL); - - xchg(&i915->gpu_error.first_error, error); - - mutex_lock(&i915->drm.struct_mutex); - - if (rq->fence.error != -EIO) { - pr_err("Guilty request not identified!\n"); - err = -EINVAL; - goto err_request; - } - -err_request: - i915_request_put(rq); -err_fini: - hang_fini(&h); -err_unlock: - mutex_unlock(&i915->drm.struct_mutex); - return err; -} - -static void __preempt_begin(void) -{ - preempt_disable(); -} - -static void __preempt_end(void) -{ - preempt_enable(); -} - -static void __softirq_begin(void) -{ - local_bh_disable(); -} - -static void __softirq_end(void) -{ - local_bh_enable(); -} - -static void __hardirq_begin(void) -{ - local_irq_disable(); -} - -static void __hardirq_end(void) -{ - local_irq_enable(); -} - -struct atomic_section { - const char *name; - void (*critical_section_begin)(void); - void (*critical_section_end)(void); -}; - -static int __igt_atomic_reset_engine(struct intel_engine_cs *engine, - const struct atomic_section *p, - const char *mode) -{ - struct tasklet_struct * const t = &engine->execlists.tasklet; - int err; - - GEM_TRACE("i915_reset_engine(%s:%s) under %s\n", - engine->name, mode, p->name); - - tasklet_disable_nosync(t); - p->critical_section_begin(); - - err = i915_reset_engine(engine, NULL); - - p->critical_section_end(); - tasklet_enable(t); - - if (err) - pr_err("i915_reset_engine(%s:%s) failed under %s\n", - engine->name, mode, p->name); - - return err; -} - -static int igt_atomic_reset_engine(struct intel_engine_cs *engine, - const struct atomic_section *p) -{ - struct drm_i915_private *i915 = engine->i915; - struct i915_request *rq; - struct hang h; - int err; - - err = __igt_atomic_reset_engine(engine, p, "idle"); - if (err) - return err; - - err = hang_init(&h, i915); - if (err) - return err; - - rq = hang_create_request(&h, engine); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto out; - } - - i915_request_get(rq); - i915_request_add(rq); - - if (wait_until_running(&h, rq)) { - err = __igt_atomic_reset_engine(engine, p, "active"); - } else { - pr_err("%s(%s): Failed to start request %llx, at %x\n", - __func__, engine->name, - rq->fence.seqno, hws_seqno(&h, rq)); - i915_gem_set_wedged(i915); - err = -EIO; - } - - if (err == 0) { - struct igt_wedge_me w; - - igt_wedge_on_timeout(&w, i915, HZ / 20 /* 50ms timeout*/) - i915_request_wait(rq, - I915_WAIT_LOCKED, - MAX_SCHEDULE_TIMEOUT); - if (i915_reset_failed(i915)) - err = -EIO; - } - - i915_request_put(rq); -out: - hang_fini(&h); - return err; -} - -static void force_reset(struct drm_i915_private *i915) -{ - i915_gem_set_wedged(i915); - i915_reset(i915, 0, NULL); -} - -static int igt_atomic_reset(void *arg) -{ - static const struct atomic_section phases[] = { - { "preempt", __preempt_begin, __preempt_end }, - { "softirq", __softirq_begin, __softirq_end }, - { "hardirq", __hardirq_begin, __hardirq_end }, - { } - }; - struct drm_i915_private *i915 = arg; - intel_wakeref_t wakeref; - int err = 0; - - /* Check that the resets are usable from atomic context */ - - if (USES_GUC_SUBMISSION(i915)) - return 0; /* guc is dead; long live the guc */ - - igt_global_reset_lock(i915); - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); - - /* Flush any requests before we get started and check basics */ - force_reset(i915); - if (i915_reset_failed(i915)) - goto unlock; - - if (intel_has_gpu_reset(i915)) { - const typeof(*phases) *p; - - for (p = phases; p->name; p++) { - GEM_TRACE("intel_gpu_reset under %s\n", p->name); - - p->critical_section_begin(); - err = intel_gpu_reset(i915, ALL_ENGINES); - p->critical_section_end(); - - if (err) { - pr_err("intel_gpu_reset failed under %s\n", - p->name); - goto out; - } - } - - force_reset(i915); - } - - if (intel_has_reset_engine(i915)) { - struct intel_engine_cs *engine; - enum intel_engine_id id; - - for_each_engine(engine, i915, id) { - const typeof(*phases) *p; - - for (p = phases; p->name; p++) { - err = igt_atomic_reset_engine(engine, p); - if (err) - goto out; - } - } - } - -out: - /* As we poke around the guts, do a full reset before continuing. */ - force_reset(i915); - -unlock: - intel_runtime_pm_put(i915, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - igt_global_reset_unlock(i915); - - return err; -} - -int intel_hangcheck_live_selftests(struct drm_i915_private *i915) -{ - static const struct i915_subtest tests[] = { - SUBTEST(igt_global_reset), /* attempt to recover GPU first */ - SUBTEST(igt_wedged_reset), - SUBTEST(igt_hang_sanitycheck), - SUBTEST(igt_reset_nop), - SUBTEST(igt_reset_nop_engine), - SUBTEST(igt_reset_idle_engine), - SUBTEST(igt_reset_active_engine), - SUBTEST(igt_reset_engines), - SUBTEST(igt_reset_queue), - SUBTEST(igt_reset_wait), - SUBTEST(igt_reset_evict_ggtt), - SUBTEST(igt_reset_evict_ppgtt), - SUBTEST(igt_reset_evict_fence), - SUBTEST(igt_handle_error), - SUBTEST(igt_atomic_reset), - }; - intel_wakeref_t wakeref; - bool saved_hangcheck; - int err; - - if (!intel_has_gpu_reset(i915)) - return 0; - - if (i915_terminally_wedged(i915)) - return -EIO; /* we're long past hope of a successful reset */ - - wakeref = intel_runtime_pm_get(i915); - saved_hangcheck = fetch_and_zero(&i915_modparams.enable_hangcheck); - drain_delayed_work(&i915->gpu_error.hangcheck_work); /* flush param */ - - err = i915_subtests(tests, i915); - - mutex_lock(&i915->drm.struct_mutex); - igt_flush_test(i915, I915_WAIT_LOCKED); - mutex_unlock(&i915->drm.struct_mutex); - - i915_modparams.enable_hangcheck = saved_hangcheck; - intel_runtime_pm_put(i915, wakeref); - - return err; -} diff --git a/drivers/gpu/drm/i915/selftests/intel_lrc.c b/drivers/gpu/drm/i915/selftests/intel_lrc.c deleted file mode 100644 index fbee030db940..000000000000 --- a/drivers/gpu/drm/i915/selftests/intel_lrc.c +++ /dev/null @@ -1,1326 +0,0 @@ -/* - * SPDX-License-Identifier: MIT - * - * Copyright © 2018 Intel Corporation - */ - -#include <linux/prime_numbers.h> - -#include "../i915_reset.h" - -#include "../i915_selftest.h" -#include "igt_flush_test.h" -#include "igt_live_test.h" -#include "igt_spinner.h" -#include "i915_random.h" - -#include "mock_context.h" - -static int live_sanitycheck(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *engine; - struct i915_gem_context *ctx; - enum intel_engine_id id; - struct igt_spinner spin; - intel_wakeref_t wakeref; - int err = -ENOMEM; - - if (!HAS_LOGICAL_RING_CONTEXTS(i915)) - return 0; - - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); - - if (igt_spinner_init(&spin, i915)) - goto err_unlock; - - ctx = kernel_context(i915); - if (!ctx) - goto err_spin; - - for_each_engine(engine, i915, id) { - struct i915_request *rq; - - rq = igt_spinner_create_request(&spin, ctx, engine, MI_NOOP); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_ctx; - } - - i915_request_add(rq); - if (!igt_wait_for_spinner(&spin, rq)) { - GEM_TRACE("spinner failed to start\n"); - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_ctx; - } - - igt_spinner_end(&spin); - if (igt_flush_test(i915, I915_WAIT_LOCKED)) { - err = -EIO; - goto err_ctx; - } - } - - err = 0; -err_ctx: - kernel_context_close(ctx); -err_spin: - igt_spinner_fini(&spin); -err_unlock: - igt_flush_test(i915, I915_WAIT_LOCKED); - intel_runtime_pm_put(i915, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - return err; -} - -static int live_busywait_preempt(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct i915_gem_context *ctx_hi, *ctx_lo; - struct intel_engine_cs *engine; - struct drm_i915_gem_object *obj; - struct i915_vma *vma; - enum intel_engine_id id; - intel_wakeref_t wakeref; - int err = -ENOMEM; - u32 *map; - - /* - * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can - * preempt the busywaits used to synchronise between rings. - */ - - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); - - ctx_hi = kernel_context(i915); - if (!ctx_hi) - goto err_unlock; - ctx_hi->sched.priority = INT_MAX; - - ctx_lo = kernel_context(i915); - if (!ctx_lo) - goto err_ctx_hi; - ctx_lo->sched.priority = INT_MIN; - - obj = i915_gem_object_create_internal(i915, PAGE_SIZE); - if (IS_ERR(obj)) { - err = PTR_ERR(obj); - goto err_ctx_lo; - } - - map = i915_gem_object_pin_map(obj, I915_MAP_WC); - if (IS_ERR(map)) { - err = PTR_ERR(map); - goto err_obj; - } - - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto err_map; - } - - err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); - if (err) - goto err_map; - - for_each_engine(engine, i915, id) { - struct i915_request *lo, *hi; - struct igt_live_test t; - u32 *cs; - - if (!intel_engine_can_store_dword(engine)) - continue; - - if (igt_live_test_begin(&t, i915, __func__, engine->name)) { - err = -EIO; - goto err_vma; - } - - /* - * We create two requests. The low priority request - * busywaits on a semaphore (inside the ringbuffer where - * is should be preemptible) and the high priority requests - * uses a MI_STORE_DWORD_IMM to update the semaphore value - * allowing the first request to complete. If preemption - * fails, we hang instead. - */ - - lo = i915_request_alloc(engine, ctx_lo); - if (IS_ERR(lo)) { - err = PTR_ERR(lo); - goto err_vma; - } - - cs = intel_ring_begin(lo, 8); - if (IS_ERR(cs)) { - err = PTR_ERR(cs); - i915_request_add(lo); - goto err_vma; - } - - *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; - *cs++ = i915_ggtt_offset(vma); - *cs++ = 0; - *cs++ = 1; - - /* XXX Do we need a flush + invalidate here? */ - - *cs++ = MI_SEMAPHORE_WAIT | - MI_SEMAPHORE_GLOBAL_GTT | - MI_SEMAPHORE_POLL | - MI_SEMAPHORE_SAD_EQ_SDD; - *cs++ = 0; - *cs++ = i915_ggtt_offset(vma); - *cs++ = 0; - - intel_ring_advance(lo, cs); - i915_request_add(lo); - - if (wait_for(READ_ONCE(*map), 10)) { - err = -ETIMEDOUT; - goto err_vma; - } - - /* Low priority request should be busywaiting now */ - if (i915_request_wait(lo, I915_WAIT_LOCKED, 1) != -ETIME) { - pr_err("%s: Busywaiting request did not!\n", - engine->name); - err = -EIO; - goto err_vma; - } - - hi = i915_request_alloc(engine, ctx_hi); - if (IS_ERR(hi)) { - err = PTR_ERR(hi); - goto err_vma; - } - - cs = intel_ring_begin(hi, 4); - if (IS_ERR(cs)) { - err = PTR_ERR(cs); - i915_request_add(hi); - goto err_vma; - } - - *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; - *cs++ = i915_ggtt_offset(vma); - *cs++ = 0; - *cs++ = 0; - - intel_ring_advance(hi, cs); - i915_request_add(hi); - - if (i915_request_wait(lo, I915_WAIT_LOCKED, HZ / 5) < 0) { - struct drm_printer p = drm_info_printer(i915->drm.dev); - - pr_err("%s: Failed to preempt semaphore busywait!\n", - engine->name); - - intel_engine_dump(engine, &p, "%s\n", engine->name); - GEM_TRACE_DUMP(); - - i915_gem_set_wedged(i915); - err = -EIO; - goto err_vma; - } - GEM_BUG_ON(READ_ONCE(*map)); - - if (igt_live_test_end(&t)) { - err = -EIO; - goto err_vma; - } - } - - err = 0; -err_vma: - i915_vma_unpin(vma); -err_map: - i915_gem_object_unpin_map(obj); -err_obj: - i915_gem_object_put(obj); -err_ctx_lo: - kernel_context_close(ctx_lo); -err_ctx_hi: - kernel_context_close(ctx_hi); -err_unlock: - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - err = -EIO; - intel_runtime_pm_put(i915, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - return err; -} - -static int live_preempt(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct i915_gem_context *ctx_hi, *ctx_lo; - struct igt_spinner spin_hi, spin_lo; - struct intel_engine_cs *engine; - enum intel_engine_id id; - intel_wakeref_t wakeref; - int err = -ENOMEM; - - if (!HAS_LOGICAL_RING_PREEMPTION(i915)) - return 0; - - if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION)) - pr_err("Logical preemption supported, but not exposed\n"); - - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); - - if (igt_spinner_init(&spin_hi, i915)) - goto err_unlock; - - if (igt_spinner_init(&spin_lo, i915)) - goto err_spin_hi; - - ctx_hi = kernel_context(i915); - if (!ctx_hi) - goto err_spin_lo; - ctx_hi->sched.priority = - I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY); - - ctx_lo = kernel_context(i915); - if (!ctx_lo) - goto err_ctx_hi; - ctx_lo->sched.priority = - I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY); - - for_each_engine(engine, i915, id) { - struct igt_live_test t; - struct i915_request *rq; - - if (!intel_engine_has_preemption(engine)) - continue; - - if (igt_live_test_begin(&t, i915, __func__, engine->name)) { - err = -EIO; - goto err_ctx_lo; - } - - rq = igt_spinner_create_request(&spin_lo, ctx_lo, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_ctx_lo; - } - - i915_request_add(rq); - if (!igt_wait_for_spinner(&spin_lo, rq)) { - GEM_TRACE("lo spinner failed to start\n"); - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_ctx_lo; - } - - rq = igt_spinner_create_request(&spin_hi, ctx_hi, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) { - igt_spinner_end(&spin_lo); - err = PTR_ERR(rq); - goto err_ctx_lo; - } - - i915_request_add(rq); - if (!igt_wait_for_spinner(&spin_hi, rq)) { - GEM_TRACE("hi spinner failed to start\n"); - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_ctx_lo; - } - - igt_spinner_end(&spin_hi); - igt_spinner_end(&spin_lo); - - if (igt_live_test_end(&t)) { - err = -EIO; - goto err_ctx_lo; - } - } - - err = 0; -err_ctx_lo: - kernel_context_close(ctx_lo); -err_ctx_hi: - kernel_context_close(ctx_hi); -err_spin_lo: - igt_spinner_fini(&spin_lo); -err_spin_hi: - igt_spinner_fini(&spin_hi); -err_unlock: - igt_flush_test(i915, I915_WAIT_LOCKED); - intel_runtime_pm_put(i915, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - return err; -} - -static int live_late_preempt(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct i915_gem_context *ctx_hi, *ctx_lo; - struct igt_spinner spin_hi, spin_lo; - struct intel_engine_cs *engine; - struct i915_sched_attr attr = {}; - enum intel_engine_id id; - intel_wakeref_t wakeref; - int err = -ENOMEM; - - if (!HAS_LOGICAL_RING_PREEMPTION(i915)) - return 0; - - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); - - if (igt_spinner_init(&spin_hi, i915)) - goto err_unlock; - - if (igt_spinner_init(&spin_lo, i915)) - goto err_spin_hi; - - ctx_hi = kernel_context(i915); - if (!ctx_hi) - goto err_spin_lo; - - ctx_lo = kernel_context(i915); - if (!ctx_lo) - goto err_ctx_hi; - - for_each_engine(engine, i915, id) { - struct igt_live_test t; - struct i915_request *rq; - - if (!intel_engine_has_preemption(engine)) - continue; - - if (igt_live_test_begin(&t, i915, __func__, engine->name)) { - err = -EIO; - goto err_ctx_lo; - } - - rq = igt_spinner_create_request(&spin_lo, ctx_lo, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_ctx_lo; - } - - i915_request_add(rq); - if (!igt_wait_for_spinner(&spin_lo, rq)) { - pr_err("First context failed to start\n"); - goto err_wedged; - } - - rq = igt_spinner_create_request(&spin_hi, ctx_hi, engine, - MI_NOOP); - if (IS_ERR(rq)) { - igt_spinner_end(&spin_lo); - err = PTR_ERR(rq); - goto err_ctx_lo; - } - - i915_request_add(rq); - if (igt_wait_for_spinner(&spin_hi, rq)) { - pr_err("Second context overtook first?\n"); - goto err_wedged; - } - - attr.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX); - engine->schedule(rq, &attr); - - if (!igt_wait_for_spinner(&spin_hi, rq)) { - pr_err("High priority context failed to preempt the low priority context\n"); - GEM_TRACE_DUMP(); - goto err_wedged; - } - - igt_spinner_end(&spin_hi); - igt_spinner_end(&spin_lo); - - if (igt_live_test_end(&t)) { - err = -EIO; - goto err_ctx_lo; - } - } - - err = 0; -err_ctx_lo: - kernel_context_close(ctx_lo); -err_ctx_hi: - kernel_context_close(ctx_hi); -err_spin_lo: - igt_spinner_fini(&spin_lo); -err_spin_hi: - igt_spinner_fini(&spin_hi); -err_unlock: - igt_flush_test(i915, I915_WAIT_LOCKED); - intel_runtime_pm_put(i915, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - return err; - -err_wedged: - igt_spinner_end(&spin_hi); - igt_spinner_end(&spin_lo); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_ctx_lo; -} - -struct preempt_client { - struct igt_spinner spin; - struct i915_gem_context *ctx; -}; - -static int preempt_client_init(struct drm_i915_private *i915, - struct preempt_client *c) -{ - c->ctx = kernel_context(i915); - if (!c->ctx) - return -ENOMEM; - - if (igt_spinner_init(&c->spin, i915)) - goto err_ctx; - - return 0; - -err_ctx: - kernel_context_close(c->ctx); - return -ENOMEM; -} - -static void preempt_client_fini(struct preempt_client *c) -{ - igt_spinner_fini(&c->spin); - kernel_context_close(c->ctx); -} - -static int live_suppress_self_preempt(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *engine; - struct i915_sched_attr attr = { - .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX) - }; - struct preempt_client a, b; - enum intel_engine_id id; - intel_wakeref_t wakeref; - int err = -ENOMEM; - - /* - * Verify that if a preemption request does not cause a change in - * the current execution order, the preempt-to-idle injection is - * skipped and that we do not accidentally apply it after the CS - * completion event. - */ - - if (!HAS_LOGICAL_RING_PREEMPTION(i915)) - return 0; - - if (USES_GUC_SUBMISSION(i915)) - return 0; /* presume black blox */ - - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); - - if (preempt_client_init(i915, &a)) - goto err_unlock; - if (preempt_client_init(i915, &b)) - goto err_client_a; - - for_each_engine(engine, i915, id) { - struct i915_request *rq_a, *rq_b; - int depth; - - if (!intel_engine_has_preemption(engine)) - continue; - - engine->execlists.preempt_hang.count = 0; - - rq_a = igt_spinner_create_request(&a.spin, - a.ctx, engine, - MI_NOOP); - if (IS_ERR(rq_a)) { - err = PTR_ERR(rq_a); - goto err_client_b; - } - - i915_request_add(rq_a); - if (!igt_wait_for_spinner(&a.spin, rq_a)) { - pr_err("First client failed to start\n"); - goto err_wedged; - } - - for (depth = 0; depth < 8; depth++) { - rq_b = igt_spinner_create_request(&b.spin, - b.ctx, engine, - MI_NOOP); - if (IS_ERR(rq_b)) { - err = PTR_ERR(rq_b); - goto err_client_b; - } - i915_request_add(rq_b); - - GEM_BUG_ON(i915_request_completed(rq_a)); - engine->schedule(rq_a, &attr); - igt_spinner_end(&a.spin); - - if (!igt_wait_for_spinner(&b.spin, rq_b)) { - pr_err("Second client failed to start\n"); - goto err_wedged; - } - - swap(a, b); - rq_a = rq_b; - } - igt_spinner_end(&a.spin); - - if (engine->execlists.preempt_hang.count) { - pr_err("Preemption recorded x%d, depth %d; should have been suppressed!\n", - engine->execlists.preempt_hang.count, - depth); - err = -EINVAL; - goto err_client_b; - } - - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - goto err_wedged; - } - - err = 0; -err_client_b: - preempt_client_fini(&b); -err_client_a: - preempt_client_fini(&a); -err_unlock: - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - err = -EIO; - intel_runtime_pm_put(i915, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - return err; - -err_wedged: - igt_spinner_end(&b.spin); - igt_spinner_end(&a.spin); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_client_b; -} - -static int __i915_sw_fence_call -dummy_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) -{ - return NOTIFY_DONE; -} - -static struct i915_request *dummy_request(struct intel_engine_cs *engine) -{ - struct i915_request *rq; - - rq = kzalloc(sizeof(*rq), GFP_KERNEL); - if (!rq) - return NULL; - - INIT_LIST_HEAD(&rq->active_list); - rq->engine = engine; - - i915_sched_node_init(&rq->sched); - - /* mark this request as permanently incomplete */ - rq->fence.seqno = 1; - BUILD_BUG_ON(sizeof(rq->fence.seqno) != 8); /* upper 32b == 0 */ - rq->hwsp_seqno = (u32 *)&rq->fence.seqno + 1; - GEM_BUG_ON(i915_request_completed(rq)); - - i915_sw_fence_init(&rq->submit, dummy_notify); - i915_sw_fence_commit(&rq->submit); - - return rq; -} - -static void dummy_request_free(struct i915_request *dummy) -{ - i915_request_mark_complete(dummy); - i915_sched_node_fini(&dummy->sched); - i915_sw_fence_fini(&dummy->submit); - - dma_fence_free(&dummy->fence); -} - -static int live_suppress_wait_preempt(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct preempt_client client[4]; - struct intel_engine_cs *engine; - enum intel_engine_id id; - intel_wakeref_t wakeref; - int err = -ENOMEM; - int i; - - /* - * Waiters are given a little priority nudge, but not enough - * to actually cause any preemption. Double check that we do - * not needlessly generate preempt-to-idle cycles. - */ - - if (!HAS_LOGICAL_RING_PREEMPTION(i915)) - return 0; - - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); - - if (preempt_client_init(i915, &client[0])) /* ELSP[0] */ - goto err_unlock; - if (preempt_client_init(i915, &client[1])) /* ELSP[1] */ - goto err_client_0; - if (preempt_client_init(i915, &client[2])) /* head of queue */ - goto err_client_1; - if (preempt_client_init(i915, &client[3])) /* bystander */ - goto err_client_2; - - for_each_engine(engine, i915, id) { - int depth; - - if (!intel_engine_has_preemption(engine)) - continue; - - if (!engine->emit_init_breadcrumb) - continue; - - for (depth = 0; depth < ARRAY_SIZE(client); depth++) { - struct i915_request *rq[ARRAY_SIZE(client)]; - struct i915_request *dummy; - - engine->execlists.preempt_hang.count = 0; - - dummy = dummy_request(engine); - if (!dummy) - goto err_client_3; - - for (i = 0; i < ARRAY_SIZE(client); i++) { - rq[i] = igt_spinner_create_request(&client[i].spin, - client[i].ctx, engine, - MI_NOOP); - if (IS_ERR(rq[i])) { - err = PTR_ERR(rq[i]); - goto err_wedged; - } - - /* Disable NEWCLIENT promotion */ - __i915_active_request_set(&rq[i]->timeline->last_request, - dummy); - i915_request_add(rq[i]); - } - - dummy_request_free(dummy); - - GEM_BUG_ON(i915_request_completed(rq[0])); - if (!igt_wait_for_spinner(&client[0].spin, rq[0])) { - pr_err("%s: First client failed to start\n", - engine->name); - goto err_wedged; - } - GEM_BUG_ON(!i915_request_started(rq[0])); - - if (i915_request_wait(rq[depth], - I915_WAIT_LOCKED | - I915_WAIT_PRIORITY, - 1) != -ETIME) { - pr_err("%s: Waiter depth:%d completed!\n", - engine->name, depth); - goto err_wedged; - } - - for (i = 0; i < ARRAY_SIZE(client); i++) - igt_spinner_end(&client[i].spin); - - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - goto err_wedged; - - if (engine->execlists.preempt_hang.count) { - pr_err("%s: Preemption recorded x%d, depth %d; should have been suppressed!\n", - engine->name, - engine->execlists.preempt_hang.count, - depth); - err = -EINVAL; - goto err_client_3; - } - } - } - - err = 0; -err_client_3: - preempt_client_fini(&client[3]); -err_client_2: - preempt_client_fini(&client[2]); -err_client_1: - preempt_client_fini(&client[1]); -err_client_0: - preempt_client_fini(&client[0]); -err_unlock: - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - err = -EIO; - intel_runtime_pm_put(i915, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - return err; - -err_wedged: - for (i = 0; i < ARRAY_SIZE(client); i++) - igt_spinner_end(&client[i].spin); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_client_3; -} - -static int live_chain_preempt(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *engine; - struct preempt_client hi, lo; - enum intel_engine_id id; - intel_wakeref_t wakeref; - int err = -ENOMEM; - - /* - * Build a chain AB...BA between two contexts (A, B) and request - * preemption of the last request. It should then complete before - * the previously submitted spinner in B. - */ - - if (!HAS_LOGICAL_RING_PREEMPTION(i915)) - return 0; - - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); - - if (preempt_client_init(i915, &hi)) - goto err_unlock; - - if (preempt_client_init(i915, &lo)) - goto err_client_hi; - - for_each_engine(engine, i915, id) { - struct i915_sched_attr attr = { - .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX), - }; - struct igt_live_test t; - struct i915_request *rq; - int ring_size, count, i; - - if (!intel_engine_has_preemption(engine)) - continue; - - rq = igt_spinner_create_request(&lo.spin, - lo.ctx, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) - goto err_wedged; - i915_request_add(rq); - - ring_size = rq->wa_tail - rq->head; - if (ring_size < 0) - ring_size += rq->ring->size; - ring_size = rq->ring->size / ring_size; - pr_debug("%s(%s): Using maximum of %d requests\n", - __func__, engine->name, ring_size); - - igt_spinner_end(&lo.spin); - if (i915_request_wait(rq, I915_WAIT_LOCKED, HZ / 2) < 0) { - pr_err("Timed out waiting to flush %s\n", engine->name); - goto err_wedged; - } - - if (igt_live_test_begin(&t, i915, __func__, engine->name)) { - err = -EIO; - goto err_wedged; - } - - for_each_prime_number_from(count, 1, ring_size) { - rq = igt_spinner_create_request(&hi.spin, - hi.ctx, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) - goto err_wedged; - i915_request_add(rq); - if (!igt_wait_for_spinner(&hi.spin, rq)) - goto err_wedged; - - rq = igt_spinner_create_request(&lo.spin, - lo.ctx, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) - goto err_wedged; - i915_request_add(rq); - - for (i = 0; i < count; i++) { - rq = i915_request_alloc(engine, lo.ctx); - if (IS_ERR(rq)) - goto err_wedged; - i915_request_add(rq); - } - - rq = i915_request_alloc(engine, hi.ctx); - if (IS_ERR(rq)) - goto err_wedged; - i915_request_add(rq); - engine->schedule(rq, &attr); - - igt_spinner_end(&hi.spin); - if (i915_request_wait(rq, I915_WAIT_LOCKED, HZ / 5) < 0) { - struct drm_printer p = - drm_info_printer(i915->drm.dev); - - pr_err("Failed to preempt over chain of %d\n", - count); - intel_engine_dump(engine, &p, - "%s\n", engine->name); - goto err_wedged; - } - igt_spinner_end(&lo.spin); - - rq = i915_request_alloc(engine, lo.ctx); - if (IS_ERR(rq)) - goto err_wedged; - i915_request_add(rq); - if (i915_request_wait(rq, I915_WAIT_LOCKED, HZ / 5) < 0) { - struct drm_printer p = - drm_info_printer(i915->drm.dev); - - pr_err("Failed to flush low priority chain of %d requests\n", - count); - intel_engine_dump(engine, &p, - "%s\n", engine->name); - goto err_wedged; - } - } - - if (igt_live_test_end(&t)) { - err = -EIO; - goto err_wedged; - } - } - - err = 0; -err_client_lo: - preempt_client_fini(&lo); -err_client_hi: - preempt_client_fini(&hi); -err_unlock: - if (igt_flush_test(i915, I915_WAIT_LOCKED)) - err = -EIO; - intel_runtime_pm_put(i915, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - return err; - -err_wedged: - igt_spinner_end(&hi.spin); - igt_spinner_end(&lo.spin); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_client_lo; -} - -static int live_preempt_hang(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct i915_gem_context *ctx_hi, *ctx_lo; - struct igt_spinner spin_hi, spin_lo; - struct intel_engine_cs *engine; - enum intel_engine_id id; - intel_wakeref_t wakeref; - int err = -ENOMEM; - - if (!HAS_LOGICAL_RING_PREEMPTION(i915)) - return 0; - - if (!intel_has_reset_engine(i915)) - return 0; - - mutex_lock(&i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(i915); - - if (igt_spinner_init(&spin_hi, i915)) - goto err_unlock; - - if (igt_spinner_init(&spin_lo, i915)) - goto err_spin_hi; - - ctx_hi = kernel_context(i915); - if (!ctx_hi) - goto err_spin_lo; - ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY; - - ctx_lo = kernel_context(i915); - if (!ctx_lo) - goto err_ctx_hi; - ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY; - - for_each_engine(engine, i915, id) { - struct i915_request *rq; - - if (!intel_engine_has_preemption(engine)) - continue; - - rq = igt_spinner_create_request(&spin_lo, ctx_lo, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_ctx_lo; - } - - i915_request_add(rq); - if (!igt_wait_for_spinner(&spin_lo, rq)) { - GEM_TRACE("lo spinner failed to start\n"); - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_ctx_lo; - } - - rq = igt_spinner_create_request(&spin_hi, ctx_hi, engine, - MI_ARB_CHECK); - if (IS_ERR(rq)) { - igt_spinner_end(&spin_lo); - err = PTR_ERR(rq); - goto err_ctx_lo; - } - - init_completion(&engine->execlists.preempt_hang.completion); - engine->execlists.preempt_hang.inject_hang = true; - - i915_request_add(rq); - - if (!wait_for_completion_timeout(&engine->execlists.preempt_hang.completion, - HZ / 10)) { - pr_err("Preemption did not occur within timeout!"); - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_ctx_lo; - } - - set_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); - i915_reset_engine(engine, NULL); - clear_bit(I915_RESET_ENGINE + id, &i915->gpu_error.flags); - - engine->execlists.preempt_hang.inject_hang = false; - - if (!igt_wait_for_spinner(&spin_hi, rq)) { - GEM_TRACE("hi spinner failed to start\n"); - GEM_TRACE_DUMP(); - i915_gem_set_wedged(i915); - err = -EIO; - goto err_ctx_lo; - } - - igt_spinner_end(&spin_hi); - igt_spinner_end(&spin_lo); - if (igt_flush_test(i915, I915_WAIT_LOCKED)) { - err = -EIO; - goto err_ctx_lo; - } - } - - err = 0; -err_ctx_lo: - kernel_context_close(ctx_lo); -err_ctx_hi: - kernel_context_close(ctx_hi); -err_spin_lo: - igt_spinner_fini(&spin_lo); -err_spin_hi: - igt_spinner_fini(&spin_hi); -err_unlock: - igt_flush_test(i915, I915_WAIT_LOCKED); - intel_runtime_pm_put(i915, wakeref); - mutex_unlock(&i915->drm.struct_mutex); - return err; -} - -static int random_range(struct rnd_state *rnd, int min, int max) -{ - return i915_prandom_u32_max_state(max - min, rnd) + min; -} - -static int random_priority(struct rnd_state *rnd) -{ - return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX); -} - -struct preempt_smoke { - struct drm_i915_private *i915; - struct i915_gem_context **contexts; - struct intel_engine_cs *engine; - struct drm_i915_gem_object *batch; - unsigned int ncontext; - struct rnd_state prng; - unsigned long count; -}; - -static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke) -{ - return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext, - &smoke->prng)]; -} - -static int smoke_submit(struct preempt_smoke *smoke, - struct i915_gem_context *ctx, int prio, - struct drm_i915_gem_object *batch) -{ - struct i915_request *rq; - struct i915_vma *vma = NULL; - int err = 0; - - if (batch) { - vma = i915_vma_instance(batch, &ctx->ppgtt->vm, NULL); - if (IS_ERR(vma)) - return PTR_ERR(vma); - - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) - return err; - } - - ctx->sched.priority = prio; - - rq = i915_request_alloc(smoke->engine, ctx); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto unpin; - } - - if (vma) { - err = rq->engine->emit_bb_start(rq, - vma->node.start, - PAGE_SIZE, 0); - if (!err) - err = i915_vma_move_to_active(vma, rq, 0); - } - - i915_request_add(rq); - -unpin: - if (vma) - i915_vma_unpin(vma); - - return err; -} - -static int smoke_crescendo_thread(void *arg) -{ - struct preempt_smoke *smoke = arg; - IGT_TIMEOUT(end_time); - unsigned long count; - - count = 0; - do { - struct i915_gem_context *ctx = smoke_context(smoke); - int err; - - mutex_lock(&smoke->i915->drm.struct_mutex); - err = smoke_submit(smoke, - ctx, count % I915_PRIORITY_MAX, - smoke->batch); - mutex_unlock(&smoke->i915->drm.struct_mutex); - if (err) - return err; - - count++; - } while (!__igt_timeout(end_time, NULL)); - - smoke->count = count; - return 0; -} - -static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags) -#define BATCH BIT(0) -{ - struct task_struct *tsk[I915_NUM_ENGINES] = {}; - struct preempt_smoke arg[I915_NUM_ENGINES]; - struct intel_engine_cs *engine; - enum intel_engine_id id; - unsigned long count; - int err = 0; - - mutex_unlock(&smoke->i915->drm.struct_mutex); - - for_each_engine(engine, smoke->i915, id) { - arg[id] = *smoke; - arg[id].engine = engine; - if (!(flags & BATCH)) - arg[id].batch = NULL; - arg[id].count = 0; - - tsk[id] = kthread_run(smoke_crescendo_thread, &arg, - "igt/smoke:%d", id); - if (IS_ERR(tsk[id])) { - err = PTR_ERR(tsk[id]); - break; - } - get_task_struct(tsk[id]); - } - - count = 0; - for_each_engine(engine, smoke->i915, id) { - int status; - - if (IS_ERR_OR_NULL(tsk[id])) - continue; - - status = kthread_stop(tsk[id]); - if (status && !err) - err = status; - - count += arg[id].count; - - put_task_struct(tsk[id]); - } - - mutex_lock(&smoke->i915->drm.struct_mutex); - - pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n", - count, flags, - RUNTIME_INFO(smoke->i915)->num_engines, smoke->ncontext); - return 0; -} - -static int smoke_random(struct preempt_smoke *smoke, unsigned int flags) -{ - enum intel_engine_id id; - IGT_TIMEOUT(end_time); - unsigned long count; - - count = 0; - do { - for_each_engine(smoke->engine, smoke->i915, id) { - struct i915_gem_context *ctx = smoke_context(smoke); - int err; - - err = smoke_submit(smoke, - ctx, random_priority(&smoke->prng), - flags & BATCH ? smoke->batch : NULL); - if (err) - return err; - - count++; - } - } while (!__igt_timeout(end_time, NULL)); - - pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n", - count, flags, - RUNTIME_INFO(smoke->i915)->num_engines, smoke->ncontext); - return 0; -} - -static int live_preempt_smoke(void *arg) -{ - struct preempt_smoke smoke = { - .i915 = arg, - .prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed), - .ncontext = 1024, - }; - const unsigned int phase[] = { 0, BATCH }; - intel_wakeref_t wakeref; - struct igt_live_test t; - int err = -ENOMEM; - u32 *cs; - int n; - - if (!HAS_LOGICAL_RING_PREEMPTION(smoke.i915)) - return 0; - - smoke.contexts = kmalloc_array(smoke.ncontext, - sizeof(*smoke.contexts), - GFP_KERNEL); - if (!smoke.contexts) - return -ENOMEM; - - mutex_lock(&smoke.i915->drm.struct_mutex); - wakeref = intel_runtime_pm_get(smoke.i915); - - smoke.batch = i915_gem_object_create_internal(smoke.i915, PAGE_SIZE); - if (IS_ERR(smoke.batch)) { - err = PTR_ERR(smoke.batch); - goto err_unlock; - } - - cs = i915_gem_object_pin_map(smoke.batch, I915_MAP_WB); - if (IS_ERR(cs)) { - err = PTR_ERR(cs); - goto err_batch; - } - for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++) - cs[n] = MI_ARB_CHECK; - cs[n] = MI_BATCH_BUFFER_END; - i915_gem_object_flush_map(smoke.batch); - i915_gem_object_unpin_map(smoke.batch); - - if (igt_live_test_begin(&t, smoke.i915, __func__, "all")) { - err = -EIO; - goto err_batch; - } - - for (n = 0; n < smoke.ncontext; n++) { - smoke.contexts[n] = kernel_context(smoke.i915); - if (!smoke.contexts[n]) - goto err_ctx; - } - - for (n = 0; n < ARRAY_SIZE(phase); n++) { - err = smoke_crescendo(&smoke, phase[n]); - if (err) - goto err_ctx; - - err = smoke_random(&smoke, phase[n]); - if (err) - goto err_ctx; - } - -err_ctx: - if (igt_live_test_end(&t)) - err = -EIO; - - for (n = 0; n < smoke.ncontext; n++) { - if (!smoke.contexts[n]) - break; - kernel_context_close(smoke.contexts[n]); - } - -err_batch: - i915_gem_object_put(smoke.batch); -err_unlock: - intel_runtime_pm_put(smoke.i915, wakeref); - mutex_unlock(&smoke.i915->drm.struct_mutex); - kfree(smoke.contexts); - - return err; -} - -int intel_execlists_live_selftests(struct drm_i915_private *i915) -{ - static const struct i915_subtest tests[] = { - SUBTEST(live_sanitycheck), - SUBTEST(live_busywait_preempt), - SUBTEST(live_preempt), - SUBTEST(live_late_preempt), - SUBTEST(live_suppress_self_preempt), - SUBTEST(live_suppress_wait_preempt), - SUBTEST(live_chain_preempt), - SUBTEST(live_preempt_hang), - SUBTEST(live_preempt_smoke), - }; - - if (!HAS_EXECLISTS(i915)) - return 0; - - if (i915_terminally_wedged(i915)) - return 0; - - return i915_subtests(tests, i915); -} diff --git a/drivers/gpu/drm/i915/selftests/intel_uncore.c b/drivers/gpu/drm/i915/selftests/intel_uncore.c index e0d7ebecb215..86815c6072a1 100644 --- a/drivers/gpu/drm/i915/selftests/intel_uncore.c +++ b/drivers/gpu/drm/i915/selftests/intel_uncore.c @@ -176,7 +176,7 @@ static int live_forcewake_ops(void *arg) return 0; } - wakeref = intel_runtime_pm_get(i915); + wakeref = intel_runtime_pm_get(&i915->runtime_pm); for_each_fw_domain(domain, uncore, tmp) { smp_store_mb(domain->active, false); @@ -247,7 +247,7 @@ static int live_forcewake_ops(void *arg) } out_rpm: - intel_runtime_pm_put(i915, wakeref); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); return err; } diff --git a/drivers/gpu/drm/i915/selftests/intel_workarounds.c b/drivers/gpu/drm/i915/selftests/intel_workarounds.c deleted file mode 100644 index 567b6f8dae86..000000000000 --- a/drivers/gpu/drm/i915/selftests/intel_workarounds.c +++ /dev/null @@ -1,901 +0,0 @@ -/* - * SPDX-License-Identifier: MIT - * - * Copyright © 2018 Intel Corporation - */ - -#include "../i915_selftest.h" -#include "../i915_reset.h" - -#include "igt_flush_test.h" -#include "igt_reset.h" -#include "igt_spinner.h" -#include "igt_wedge_me.h" -#include "mock_context.h" -#include "mock_drm.h" - -static const struct wo_register { - enum intel_platform platform; - u32 reg; -} wo_registers[] = { - { INTEL_GEMINILAKE, 0x731c } -}; - -#define REF_NAME_MAX (INTEL_ENGINE_CS_MAX_NAME + 4) -struct wa_lists { - struct i915_wa_list gt_wa_list; - struct { - char name[REF_NAME_MAX]; - struct i915_wa_list wa_list; - } engine[I915_NUM_ENGINES]; -}; - -static void -reference_lists_init(struct drm_i915_private *i915, struct wa_lists *lists) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - memset(lists, 0, sizeof(*lists)); - - wa_init_start(&lists->gt_wa_list, "GT_REF"); - gt_init_workarounds(i915, &lists->gt_wa_list); - wa_init_finish(&lists->gt_wa_list); - - for_each_engine(engine, i915, id) { - struct i915_wa_list *wal = &lists->engine[id].wa_list; - char *name = lists->engine[id].name; - - snprintf(name, REF_NAME_MAX, "%s_REF", engine->name); - - wa_init_start(wal, name); - engine_init_workarounds(engine, wal); - wa_init_finish(wal); - } -} - -static void -reference_lists_fini(struct drm_i915_private *i915, struct wa_lists *lists) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - - for_each_engine(engine, i915, id) - intel_wa_list_free(&lists->engine[id].wa_list); - - intel_wa_list_free(&lists->gt_wa_list); -} - -static struct drm_i915_gem_object * -read_nonprivs(struct i915_gem_context *ctx, struct intel_engine_cs *engine) -{ - const u32 base = engine->mmio_base; - struct drm_i915_gem_object *result; - intel_wakeref_t wakeref; - struct i915_request *rq; - struct i915_vma *vma; - u32 srm, *cs; - int err; - int i; - - result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); - if (IS_ERR(result)) - return result; - - i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC); - - cs = i915_gem_object_pin_map(result, I915_MAP_WB); - if (IS_ERR(cs)) { - err = PTR_ERR(cs); - goto err_obj; - } - memset(cs, 0xc5, PAGE_SIZE); - i915_gem_object_flush_map(result); - i915_gem_object_unpin_map(result); - - vma = i915_vma_instance(result, &engine->i915->ggtt.vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto err_obj; - } - - err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); - if (err) - goto err_obj; - - rq = ERR_PTR(-ENODEV); - with_intel_runtime_pm(engine->i915, wakeref) - rq = i915_request_alloc(engine, ctx); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto err_pin; - } - - err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); - if (err) - goto err_req; - - srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; - if (INTEL_GEN(ctx->i915) >= 8) - srm++; - - cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS); - if (IS_ERR(cs)) { - err = PTR_ERR(cs); - goto err_req; - } - - for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { - *cs++ = srm; - *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i)); - *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i; - *cs++ = 0; - } - intel_ring_advance(rq, cs); - - i915_gem_object_get(result); - i915_gem_object_set_active_reference(result); - - i915_request_add(rq); - i915_vma_unpin(vma); - - return result; - -err_req: - i915_request_add(rq); -err_pin: - i915_vma_unpin(vma); -err_obj: - i915_gem_object_put(result); - return ERR_PTR(err); -} - -static u32 -get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i) -{ - i915_reg_t reg = i < engine->whitelist.count ? - engine->whitelist.list[i].reg : - RING_NOPID(engine->mmio_base); - - return i915_mmio_reg_offset(reg); -} - -static void -print_results(const struct intel_engine_cs *engine, const u32 *results) -{ - unsigned int i; - - for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { - u32 expected = get_whitelist_reg(engine, i); - u32 actual = results[i]; - - pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n", - i, expected, actual); - } -} - -static int check_whitelist(struct i915_gem_context *ctx, - struct intel_engine_cs *engine) -{ - struct drm_i915_gem_object *results; - struct igt_wedge_me wedge; - u32 *vaddr; - int err; - int i; - - results = read_nonprivs(ctx, engine); - if (IS_ERR(results)) - return PTR_ERR(results); - - err = 0; - igt_wedge_on_timeout(&wedge, ctx->i915, HZ / 5) /* a safety net! */ - err = i915_gem_object_set_to_cpu_domain(results, false); - if (i915_terminally_wedged(ctx->i915)) - err = -EIO; - if (err) - goto out_put; - - vaddr = i915_gem_object_pin_map(results, I915_MAP_WB); - if (IS_ERR(vaddr)) { - err = PTR_ERR(vaddr); - goto out_put; - } - - for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { - u32 expected = get_whitelist_reg(engine, i); - u32 actual = vaddr[i]; - - if (expected != actual) { - print_results(engine, vaddr); - pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n", - i, expected, actual); - - err = -EINVAL; - break; - } - } - - i915_gem_object_unpin_map(results); -out_put: - i915_gem_object_put(results); - return err; -} - -static int do_device_reset(struct intel_engine_cs *engine) -{ - i915_reset(engine->i915, engine->mask, "live_workarounds"); - return 0; -} - -static int do_engine_reset(struct intel_engine_cs *engine) -{ - return i915_reset_engine(engine, "live_workarounds"); -} - -static int -switch_to_scratch_context(struct intel_engine_cs *engine, - struct igt_spinner *spin) -{ - struct i915_gem_context *ctx; - struct i915_request *rq; - intel_wakeref_t wakeref; - int err = 0; - - ctx = kernel_context(engine->i915); - if (IS_ERR(ctx)) - return PTR_ERR(ctx); - - GEM_BUG_ON(i915_gem_context_is_bannable(ctx)); - - rq = ERR_PTR(-ENODEV); - with_intel_runtime_pm(engine->i915, wakeref) - rq = igt_spinner_create_request(spin, ctx, engine, MI_NOOP); - - kernel_context_close(ctx); - - if (IS_ERR(rq)) { - spin = NULL; - err = PTR_ERR(rq); - goto err; - } - - i915_request_add(rq); - - if (spin && !igt_wait_for_spinner(spin, rq)) { - pr_err("Spinner failed to start\n"); - err = -ETIMEDOUT; - } - -err: - if (err && spin) - igt_spinner_end(spin); - - return err; -} - -static int check_whitelist_across_reset(struct intel_engine_cs *engine, - int (*reset)(struct intel_engine_cs *), - const char *name) -{ - struct drm_i915_private *i915 = engine->i915; - struct i915_gem_context *ctx; - struct igt_spinner spin; - intel_wakeref_t wakeref; - int err; - - pr_info("Checking %d whitelisted registers (RING_NONPRIV) [%s]\n", - engine->whitelist.count, name); - - err = igt_spinner_init(&spin, i915); - if (err) - return err; - - ctx = kernel_context(i915); - if (IS_ERR(ctx)) - return PTR_ERR(ctx); - - err = check_whitelist(ctx, engine); - if (err) { - pr_err("Invalid whitelist *before* %s reset!\n", name); - goto out; - } - - err = switch_to_scratch_context(engine, &spin); - if (err) - goto out; - - with_intel_runtime_pm(i915, wakeref) - err = reset(engine); - - igt_spinner_end(&spin); - igt_spinner_fini(&spin); - - if (err) { - pr_err("%s reset failed\n", name); - goto out; - } - - err = check_whitelist(ctx, engine); - if (err) { - pr_err("Whitelist not preserved in context across %s reset!\n", - name); - goto out; - } - - kernel_context_close(ctx); - - ctx = kernel_context(i915); - if (IS_ERR(ctx)) - return PTR_ERR(ctx); - - err = check_whitelist(ctx, engine); - if (err) { - pr_err("Invalid whitelist *after* %s reset in fresh context!\n", - name); - goto out; - } - -out: - kernel_context_close(ctx); - return err; -} - -static struct i915_vma *create_scratch(struct i915_gem_context *ctx) -{ - struct drm_i915_gem_object *obj; - struct i915_vma *vma; - void *ptr; - int err; - - obj = i915_gem_object_create_internal(ctx->i915, PAGE_SIZE); - if (IS_ERR(obj)) - return ERR_CAST(obj); - - i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC); - - ptr = i915_gem_object_pin_map(obj, I915_MAP_WB); - if (IS_ERR(ptr)) { - err = PTR_ERR(ptr); - goto err_obj; - } - memset(ptr, 0xc5, PAGE_SIZE); - i915_gem_object_flush_map(obj); - i915_gem_object_unpin_map(obj); - - vma = i915_vma_instance(obj, &ctx->ppgtt->vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto err_obj; - } - - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) - goto err_obj; - - err = i915_gem_object_set_to_cpu_domain(obj, false); - if (err) - goto err_obj; - - return vma; - -err_obj: - i915_gem_object_put(obj); - return ERR_PTR(err); -} - -static struct i915_vma *create_batch(struct i915_gem_context *ctx) -{ - struct drm_i915_gem_object *obj; - struct i915_vma *vma; - int err; - - obj = i915_gem_object_create_internal(ctx->i915, 16 * PAGE_SIZE); - if (IS_ERR(obj)) - return ERR_CAST(obj); - - vma = i915_vma_instance(obj, &ctx->ppgtt->vm, NULL); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto err_obj; - } - - err = i915_vma_pin(vma, 0, 0, PIN_USER); - if (err) - goto err_obj; - - err = i915_gem_object_set_to_wc_domain(obj, true); - if (err) - goto err_obj; - - return vma; - -err_obj: - i915_gem_object_put(obj); - return ERR_PTR(err); -} - -static u32 reg_write(u32 old, u32 new, u32 rsvd) -{ - if (rsvd == 0x0000ffff) { - old &= ~(new >> 16); - old |= new & (new >> 16); - } else { - old &= ~rsvd; - old |= new & rsvd; - } - - return old; -} - -static bool wo_register(struct intel_engine_cs *engine, u32 reg) -{ - enum intel_platform platform = INTEL_INFO(engine->i915)->platform; - int i; - - for (i = 0; i < ARRAY_SIZE(wo_registers); i++) { - if (wo_registers[i].platform == platform && - wo_registers[i].reg == reg) - return true; - } - - return false; -} - -static int check_dirty_whitelist(struct i915_gem_context *ctx, - struct intel_engine_cs *engine) -{ - const u32 values[] = { - 0x00000000, - 0x01010101, - 0x10100101, - 0x03030303, - 0x30300303, - 0x05050505, - 0x50500505, - 0x0f0f0f0f, - 0xf00ff00f, - 0x10101010, - 0xf0f01010, - 0x30303030, - 0xa0a03030, - 0x50505050, - 0xc0c05050, - 0xf0f0f0f0, - 0x11111111, - 0x33333333, - 0x55555555, - 0x0000ffff, - 0x00ff00ff, - 0xff0000ff, - 0xffff00ff, - 0xffffffff, - }; - struct i915_vma *scratch; - struct i915_vma *batch; - int err = 0, i, v; - u32 *cs, *results; - - scratch = create_scratch(ctx); - if (IS_ERR(scratch)) - return PTR_ERR(scratch); - - batch = create_batch(ctx); - if (IS_ERR(batch)) { - err = PTR_ERR(batch); - goto out_scratch; - } - - for (i = 0; i < engine->whitelist.count; i++) { - u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); - u64 addr = scratch->node.start; - struct i915_request *rq; - u32 srm, lrm, rsvd; - u32 expect; - int idx; - - if (wo_register(engine, reg)) - continue; - - srm = MI_STORE_REGISTER_MEM; - lrm = MI_LOAD_REGISTER_MEM; - if (INTEL_GEN(ctx->i915) >= 8) - lrm++, srm++; - - pr_debug("%s: Writing garbage to %x\n", - engine->name, reg); - - cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); - if (IS_ERR(cs)) { - err = PTR_ERR(cs); - goto out_batch; - } - - /* SRM original */ - *cs++ = srm; - *cs++ = reg; - *cs++ = lower_32_bits(addr); - *cs++ = upper_32_bits(addr); - - idx = 1; - for (v = 0; v < ARRAY_SIZE(values); v++) { - /* LRI garbage */ - *cs++ = MI_LOAD_REGISTER_IMM(1); - *cs++ = reg; - *cs++ = values[v]; - - /* SRM result */ - *cs++ = srm; - *cs++ = reg; - *cs++ = lower_32_bits(addr + sizeof(u32) * idx); - *cs++ = upper_32_bits(addr + sizeof(u32) * idx); - idx++; - } - for (v = 0; v < ARRAY_SIZE(values); v++) { - /* LRI garbage */ - *cs++ = MI_LOAD_REGISTER_IMM(1); - *cs++ = reg; - *cs++ = ~values[v]; - - /* SRM result */ - *cs++ = srm; - *cs++ = reg; - *cs++ = lower_32_bits(addr + sizeof(u32) * idx); - *cs++ = upper_32_bits(addr + sizeof(u32) * idx); - idx++; - } - GEM_BUG_ON(idx * sizeof(u32) > scratch->size); - - /* LRM original -- don't leave garbage in the context! */ - *cs++ = lrm; - *cs++ = reg; - *cs++ = lower_32_bits(addr); - *cs++ = upper_32_bits(addr); - - *cs++ = MI_BATCH_BUFFER_END; - - i915_gem_object_flush_map(batch->obj); - i915_gem_object_unpin_map(batch->obj); - i915_gem_chipset_flush(ctx->i915); - - rq = i915_request_alloc(engine, ctx); - if (IS_ERR(rq)) { - err = PTR_ERR(rq); - goto out_batch; - } - - if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ - err = engine->emit_init_breadcrumb(rq); - if (err) - goto err_request; - } - - err = engine->emit_bb_start(rq, - batch->node.start, PAGE_SIZE, - 0); - if (err) - goto err_request; - -err_request: - i915_request_add(rq); - if (err) - goto out_batch; - - if (i915_request_wait(rq, I915_WAIT_LOCKED, HZ / 5) < 0) { - pr_err("%s: Futzing %x timedout; cancelling test\n", - engine->name, reg); - i915_gem_set_wedged(ctx->i915); - err = -EIO; - goto out_batch; - } - - results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); - if (IS_ERR(results)) { - err = PTR_ERR(results); - goto out_batch; - } - - GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff); - rsvd = results[ARRAY_SIZE(values)]; /* detect write masking */ - if (!rsvd) { - pr_err("%s: Unable to write to whitelisted register %x\n", - engine->name, reg); - err = -EINVAL; - goto out_unpin; - } - - expect = results[0]; - idx = 1; - for (v = 0; v < ARRAY_SIZE(values); v++) { - expect = reg_write(expect, values[v], rsvd); - if (results[idx] != expect) - err++; - idx++; - } - for (v = 0; v < ARRAY_SIZE(values); v++) { - expect = reg_write(expect, ~values[v], rsvd); - if (results[idx] != expect) - err++; - idx++; - } - if (err) { - pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n", - engine->name, err, reg); - - pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n", - engine->name, reg, results[0], rsvd); - - expect = results[0]; - idx = 1; - for (v = 0; v < ARRAY_SIZE(values); v++) { - u32 w = values[v]; - - expect = reg_write(expect, w, rsvd); - pr_info("Wrote %08x, read %08x, expect %08x\n", - w, results[idx], expect); - idx++; - } - for (v = 0; v < ARRAY_SIZE(values); v++) { - u32 w = ~values[v]; - - expect = reg_write(expect, w, rsvd); - pr_info("Wrote %08x, read %08x, expect %08x\n", - w, results[idx], expect); - idx++; - } - - err = -EINVAL; - } -out_unpin: - i915_gem_object_unpin_map(scratch->obj); - if (err) - break; - } - - if (igt_flush_test(ctx->i915, I915_WAIT_LOCKED)) - err = -EIO; -out_batch: - i915_vma_unpin_and_release(&batch, 0); -out_scratch: - i915_vma_unpin_and_release(&scratch, 0); - return err; -} - -static int live_dirty_whitelist(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *engine; - struct i915_gem_context *ctx; - enum intel_engine_id id; - intel_wakeref_t wakeref; - struct drm_file *file; - int err = 0; - - /* Can the user write to the whitelisted registers? */ - - if (INTEL_GEN(i915) < 7) /* minimum requirement for LRI, SRM, LRM */ - return 0; - - wakeref = intel_runtime_pm_get(i915); - - mutex_unlock(&i915->drm.struct_mutex); - file = mock_file(i915); - mutex_lock(&i915->drm.struct_mutex); - if (IS_ERR(file)) { - err = PTR_ERR(file); - goto out_rpm; - } - - ctx = live_context(i915, file); - if (IS_ERR(ctx)) { - err = PTR_ERR(ctx); - goto out_file; - } - - for_each_engine(engine, i915, id) { - if (engine->whitelist.count == 0) - continue; - - err = check_dirty_whitelist(ctx, engine); - if (err) - goto out_file; - } - -out_file: - mutex_unlock(&i915->drm.struct_mutex); - mock_file_free(i915, file); - mutex_lock(&i915->drm.struct_mutex); -out_rpm: - intel_runtime_pm_put(i915, wakeref); - return err; -} - -static int live_reset_whitelist(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *engine = i915->engine[RCS0]; - int err = 0; - - /* If we reset the gpu, we should not lose the RING_NONPRIV */ - - if (!engine || engine->whitelist.count == 0) - return 0; - - igt_global_reset_lock(i915); - - if (intel_has_reset_engine(i915)) { - err = check_whitelist_across_reset(engine, - do_engine_reset, - "engine"); - if (err) - goto out; - } - - if (intel_has_gpu_reset(i915)) { - err = check_whitelist_across_reset(engine, - do_device_reset, - "device"); - if (err) - goto out; - } - -out: - igt_global_reset_unlock(i915); - return err; -} - -static bool verify_gt_engine_wa(struct drm_i915_private *i915, - struct wa_lists *lists, const char *str) -{ - struct intel_engine_cs *engine; - enum intel_engine_id id; - bool ok = true; - - ok &= wa_list_verify(&i915->uncore, &lists->gt_wa_list, str); - - for_each_engine(engine, i915, id) - ok &= wa_list_verify(engine->uncore, - &lists->engine[id].wa_list, str); - - return ok; -} - -static int -live_gpu_reset_gt_engine_workarounds(void *arg) -{ - struct drm_i915_private *i915 = arg; - intel_wakeref_t wakeref; - struct wa_lists lists; - bool ok; - - if (!intel_has_gpu_reset(i915)) - return 0; - - pr_info("Verifying after GPU reset...\n"); - - igt_global_reset_lock(i915); - wakeref = intel_runtime_pm_get(i915); - - reference_lists_init(i915, &lists); - - ok = verify_gt_engine_wa(i915, &lists, "before reset"); - if (!ok) - goto out; - - i915_reset(i915, ALL_ENGINES, "live_workarounds"); - - ok = verify_gt_engine_wa(i915, &lists, "after reset"); - -out: - reference_lists_fini(i915, &lists); - intel_runtime_pm_put(i915, wakeref); - igt_global_reset_unlock(i915); - - return ok ? 0 : -ESRCH; -} - -static int -live_engine_reset_gt_engine_workarounds(void *arg) -{ - struct drm_i915_private *i915 = arg; - struct intel_engine_cs *engine; - struct i915_gem_context *ctx; - struct igt_spinner spin; - enum intel_engine_id id; - struct i915_request *rq; - intel_wakeref_t wakeref; - struct wa_lists lists; - int ret = 0; - - if (!intel_has_reset_engine(i915)) - return 0; - - ctx = kernel_context(i915); - if (IS_ERR(ctx)) - return PTR_ERR(ctx); - - igt_global_reset_lock(i915); - wakeref = intel_runtime_pm_get(i915); - - reference_lists_init(i915, &lists); - - for_each_engine(engine, i915, id) { - bool ok; - - pr_info("Verifying after %s reset...\n", engine->name); - - ok = verify_gt_engine_wa(i915, &lists, "before reset"); - if (!ok) { - ret = -ESRCH; - goto err; - } - - i915_reset_engine(engine, "live_workarounds"); - - ok = verify_gt_engine_wa(i915, &lists, "after idle reset"); - if (!ok) { - ret = -ESRCH; - goto err; - } - - ret = igt_spinner_init(&spin, i915); - if (ret) - goto err; - - rq = igt_spinner_create_request(&spin, ctx, engine, MI_NOOP); - if (IS_ERR(rq)) { - ret = PTR_ERR(rq); - igt_spinner_fini(&spin); - goto err; - } - - i915_request_add(rq); - - if (!igt_wait_for_spinner(&spin, rq)) { - pr_err("Spinner failed to start\n"); - igt_spinner_fini(&spin); - ret = -ETIMEDOUT; - goto err; - } - - i915_reset_engine(engine, "live_workarounds"); - - igt_spinner_end(&spin); - igt_spinner_fini(&spin); - - ok = verify_gt_engine_wa(i915, &lists, "after busy reset"); - if (!ok) { - ret = -ESRCH; - goto err; - } - } - -err: - reference_lists_fini(i915, &lists); - intel_runtime_pm_put(i915, wakeref); - igt_global_reset_unlock(i915); - kernel_context_close(ctx); - - igt_flush_test(i915, I915_WAIT_LOCKED); - - return ret; -} - -int intel_workarounds_live_selftests(struct drm_i915_private *i915) -{ - static const struct i915_subtest tests[] = { - SUBTEST(live_dirty_whitelist), - SUBTEST(live_reset_whitelist), - SUBTEST(live_gpu_reset_gt_engine_workarounds), - SUBTEST(live_engine_reset_gt_engine_workarounds), - }; - int err; - - if (i915_terminally_wedged(i915)) - return 0; - - mutex_lock(&i915->drm.struct_mutex); - err = i915_subtests(tests, i915); - mutex_unlock(&i915->drm.struct_mutex); - - return err; -} diff --git a/drivers/gpu/drm/i915/selftests/lib_sw_fence.c b/drivers/gpu/drm/i915/selftests/lib_sw_fence.c index 2bfa72c1654b..b976c12817c5 100644 --- a/drivers/gpu/drm/i915/selftests/lib_sw_fence.c +++ b/drivers/gpu/drm/i915/selftests/lib_sw_fence.c @@ -45,6 +45,9 @@ void __onstack_fence_init(struct i915_sw_fence *fence, void onstack_fence_fini(struct i915_sw_fence *fence) { + if (!fence->flags) + return; + i915_sw_fence_commit(fence); i915_sw_fence_fini(fence); } diff --git a/drivers/gpu/drm/i915/selftests/lib_sw_fence.h b/drivers/gpu/drm/i915/selftests/lib_sw_fence.h index 1f9927e10f3a..e54d6bc23dc3 100644 --- a/drivers/gpu/drm/i915/selftests/lib_sw_fence.h +++ b/drivers/gpu/drm/i915/selftests/lib_sw_fence.h @@ -1,10 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * lib_sw_fence.h - library routines for testing N:M synchronisation points * * Copyright (C) 2017 Intel Corporation - * - * This file is released under the GPLv2. - * */ #ifndef _LIB_SW_FENCE_H_ diff --git a/drivers/gpu/drm/i915/selftests/mock_context.c b/drivers/gpu/drm/i915/selftests/mock_context.c deleted file mode 100644 index 0426093bf1d9..000000000000 --- a/drivers/gpu/drm/i915/selftests/mock_context.c +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright © 2016 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#include "mock_context.h" -#include "mock_gtt.h" - -struct i915_gem_context * -mock_context(struct drm_i915_private *i915, - const char *name) -{ - struct i915_gem_context *ctx; - int ret; - - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); - if (!ctx) - return NULL; - - kref_init(&ctx->ref); - INIT_LIST_HEAD(&ctx->link); - ctx->i915 = i915; - - ctx->hw_contexts = RB_ROOT; - spin_lock_init(&ctx->hw_contexts_lock); - - INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); - INIT_LIST_HEAD(&ctx->handles_list); - INIT_LIST_HEAD(&ctx->hw_id_link); - INIT_LIST_HEAD(&ctx->active_engines); - mutex_init(&ctx->mutex); - - ret = i915_gem_context_pin_hw_id(ctx); - if (ret < 0) - goto err_handles; - - if (name) { - struct i915_hw_ppgtt *ppgtt; - - ctx->name = kstrdup(name, GFP_KERNEL); - if (!ctx->name) - goto err_put; - - ppgtt = mock_ppgtt(i915, name); - if (!ppgtt) - goto err_put; - - __set_ppgtt(ctx, ppgtt); - } - - return ctx; - -err_handles: - kfree(ctx); - return NULL; - -err_put: - i915_gem_context_set_closed(ctx); - i915_gem_context_put(ctx); - return NULL; -} - -void mock_context_close(struct i915_gem_context *ctx) -{ - context_close(ctx); -} - -void mock_init_contexts(struct drm_i915_private *i915) -{ - init_contexts(i915); -} - -struct i915_gem_context * -live_context(struct drm_i915_private *i915, struct drm_file *file) -{ - struct i915_gem_context *ctx; - int err; - - lockdep_assert_held(&i915->drm.struct_mutex); - - ctx = i915_gem_create_context(i915, 0); - if (IS_ERR(ctx)) - return ctx; - - err = gem_context_register(ctx, file->driver_priv); - if (err < 0) - goto err_ctx; - - return ctx; - -err_ctx: - context_close(ctx); - return ERR_PTR(err); -} - -struct i915_gem_context * -kernel_context(struct drm_i915_private *i915) -{ - return i915_gem_context_create_kernel(i915, I915_PRIORITY_NORMAL); -} - -void kernel_context_close(struct i915_gem_context *ctx) -{ - context_close(ctx); -} diff --git a/drivers/gpu/drm/i915/selftests/mock_context.h b/drivers/gpu/drm/i915/selftests/mock_context.h deleted file mode 100644 index 29b9d60a158b..000000000000 --- a/drivers/gpu/drm/i915/selftests/mock_context.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright © 2016 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#ifndef __MOCK_CONTEXT_H -#define __MOCK_CONTEXT_H - -void mock_init_contexts(struct drm_i915_private *i915); - -struct i915_gem_context * -mock_context(struct drm_i915_private *i915, - const char *name); - -void mock_context_close(struct i915_gem_context *ctx); - -struct i915_gem_context * -live_context(struct drm_i915_private *i915, struct drm_file *file); - -struct i915_gem_context *kernel_context(struct drm_i915_private *i915); -void kernel_context_close(struct i915_gem_context *ctx); - -#endif /* !__MOCK_CONTEXT_H */ diff --git a/drivers/gpu/drm/i915/selftests/mock_dmabuf.c b/drivers/gpu/drm/i915/selftests/mock_dmabuf.c deleted file mode 100644 index ca682caf1062..000000000000 --- a/drivers/gpu/drm/i915/selftests/mock_dmabuf.c +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright © 2016 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#include "mock_dmabuf.h" - -static struct sg_table *mock_map_dma_buf(struct dma_buf_attachment *attachment, - enum dma_data_direction dir) -{ - struct mock_dmabuf *mock = to_mock(attachment->dmabuf); - struct sg_table *st; - struct scatterlist *sg; - int i, err; - - st = kmalloc(sizeof(*st), GFP_KERNEL); - if (!st) - return ERR_PTR(-ENOMEM); - - err = sg_alloc_table(st, mock->npages, GFP_KERNEL); - if (err) - goto err_free; - - sg = st->sgl; - for (i = 0; i < mock->npages; i++) { - sg_set_page(sg, mock->pages[i], PAGE_SIZE, 0); - sg = sg_next(sg); - } - - if (!dma_map_sg(attachment->dev, st->sgl, st->nents, dir)) { - err = -ENOMEM; - goto err_st; - } - - return st; - -err_st: - sg_free_table(st); -err_free: - kfree(st); - return ERR_PTR(err); -} - -static void mock_unmap_dma_buf(struct dma_buf_attachment *attachment, - struct sg_table *st, - enum dma_data_direction dir) -{ - dma_unmap_sg(attachment->dev, st->sgl, st->nents, dir); - sg_free_table(st); - kfree(st); -} - -static void mock_dmabuf_release(struct dma_buf *dma_buf) -{ - struct mock_dmabuf *mock = to_mock(dma_buf); - int i; - - for (i = 0; i < mock->npages; i++) - put_page(mock->pages[i]); - - kfree(mock); -} - -static void *mock_dmabuf_vmap(struct dma_buf *dma_buf) -{ - struct mock_dmabuf *mock = to_mock(dma_buf); - - return vm_map_ram(mock->pages, mock->npages, 0, PAGE_KERNEL); -} - -static void mock_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr) -{ - struct mock_dmabuf *mock = to_mock(dma_buf); - - vm_unmap_ram(vaddr, mock->npages); -} - -static void *mock_dmabuf_kmap(struct dma_buf *dma_buf, unsigned long page_num) -{ - struct mock_dmabuf *mock = to_mock(dma_buf); - - return kmap(mock->pages[page_num]); -} - -static void mock_dmabuf_kunmap(struct dma_buf *dma_buf, unsigned long page_num, void *addr) -{ - struct mock_dmabuf *mock = to_mock(dma_buf); - - return kunmap(mock->pages[page_num]); -} - -static int mock_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma) -{ - return -ENODEV; -} - -static const struct dma_buf_ops mock_dmabuf_ops = { - .map_dma_buf = mock_map_dma_buf, - .unmap_dma_buf = mock_unmap_dma_buf, - .release = mock_dmabuf_release, - .map = mock_dmabuf_kmap, - .unmap = mock_dmabuf_kunmap, - .mmap = mock_dmabuf_mmap, - .vmap = mock_dmabuf_vmap, - .vunmap = mock_dmabuf_vunmap, -}; - -static struct dma_buf *mock_dmabuf(int npages) -{ - struct mock_dmabuf *mock; - DEFINE_DMA_BUF_EXPORT_INFO(exp_info); - struct dma_buf *dmabuf; - int i; - - mock = kmalloc(sizeof(*mock) + npages * sizeof(struct page *), - GFP_KERNEL); - if (!mock) - return ERR_PTR(-ENOMEM); - - mock->npages = npages; - for (i = 0; i < npages; i++) { - mock->pages[i] = alloc_page(GFP_KERNEL); - if (!mock->pages[i]) - goto err; - } - - exp_info.ops = &mock_dmabuf_ops; - exp_info.size = npages * PAGE_SIZE; - exp_info.flags = O_CLOEXEC; - exp_info.priv = mock; - - dmabuf = dma_buf_export(&exp_info); - if (IS_ERR(dmabuf)) - goto err; - - return dmabuf; - -err: - while (i--) - put_page(mock->pages[i]); - kfree(mock); - return ERR_PTR(-ENOMEM); -} diff --git a/drivers/gpu/drm/i915/selftests/mock_dmabuf.h b/drivers/gpu/drm/i915/selftests/mock_dmabuf.h deleted file mode 100644 index ec80613159b9..000000000000 --- a/drivers/gpu/drm/i915/selftests/mock_dmabuf.h +++ /dev/null @@ -1,41 +0,0 @@ - -/* - * Copyright © 2016 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#ifndef __MOCK_DMABUF_H__ -#define __MOCK_DMABUF_H__ - -#include <linux/dma-buf.h> - -struct mock_dmabuf { - int npages; - struct page *pages[]; -}; - -static struct mock_dmabuf *to_mock(struct dma_buf *buf) -{ - return buf->priv; -} - -#endif /* !__MOCK_DMABUF_H__ */ diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.c b/drivers/gpu/drm/i915/selftests/mock_engine.c deleted file mode 100644 index 61a8206ed677..000000000000 --- a/drivers/gpu/drm/i915/selftests/mock_engine.c +++ /dev/null @@ -1,321 +0,0 @@ -/* - * Copyright © 2016 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#include "mock_engine.h" -#include "mock_request.h" - -struct mock_ring { - struct intel_ring base; - struct i915_timeline timeline; -}; - -static void mock_timeline_pin(struct i915_timeline *tl) -{ - tl->pin_count++; -} - -static void mock_timeline_unpin(struct i915_timeline *tl) -{ - GEM_BUG_ON(!tl->pin_count); - tl->pin_count--; -} - -static struct intel_ring *mock_ring(struct intel_engine_cs *engine) -{ - const unsigned long sz = PAGE_SIZE / 2; - struct mock_ring *ring; - - ring = kzalloc(sizeof(*ring) + sz, GFP_KERNEL); - if (!ring) - return NULL; - - if (i915_timeline_init(engine->i915, &ring->timeline, NULL)) { - kfree(ring); - return NULL; - } - - kref_init(&ring->base.ref); - ring->base.size = sz; - ring->base.effective_size = sz; - ring->base.vaddr = (void *)(ring + 1); - ring->base.timeline = &ring->timeline; - - INIT_LIST_HEAD(&ring->base.request_list); - intel_ring_update_space(&ring->base); - - return &ring->base; -} - -static void mock_ring_free(struct intel_ring *base) -{ - struct mock_ring *ring = container_of(base, typeof(*ring), base); - - i915_timeline_fini(&ring->timeline); - kfree(ring); -} - -static struct i915_request *first_request(struct mock_engine *engine) -{ - return list_first_entry_or_null(&engine->hw_queue, - struct i915_request, - mock.link); -} - -static void advance(struct i915_request *request) -{ - list_del_init(&request->mock.link); - i915_request_mark_complete(request); - GEM_BUG_ON(!i915_request_completed(request)); - - intel_engine_queue_breadcrumbs(request->engine); -} - -static void hw_delay_complete(struct timer_list *t) -{ - struct mock_engine *engine = from_timer(engine, t, hw_delay); - struct i915_request *request; - unsigned long flags; - - spin_lock_irqsave(&engine->hw_lock, flags); - - /* Timer fired, first request is complete */ - request = first_request(engine); - if (request) - advance(request); - - /* - * Also immediately signal any subsequent 0-delay requests, but - * requeue the timer for the next delayed request. - */ - while ((request = first_request(engine))) { - if (request->mock.delay) { - mod_timer(&engine->hw_delay, - jiffies + request->mock.delay); - break; - } - - advance(request); - } - - spin_unlock_irqrestore(&engine->hw_lock, flags); -} - -static void mock_context_unpin(struct intel_context *ce) -{ - mock_timeline_unpin(ce->ring->timeline); -} - -static void mock_context_destroy(struct kref *ref) -{ - struct intel_context *ce = container_of(ref, typeof(*ce), ref); - - GEM_BUG_ON(intel_context_is_pinned(ce)); - - if (ce->ring) - mock_ring_free(ce->ring); - - intel_context_free(ce); -} - -static int mock_context_pin(struct intel_context *ce) -{ - if (!ce->ring) { - ce->ring = mock_ring(ce->engine); - if (!ce->ring) - return -ENOMEM; - } - - mock_timeline_pin(ce->ring->timeline); - return 0; -} - -static const struct intel_context_ops mock_context_ops = { - .pin = mock_context_pin, - .unpin = mock_context_unpin, - - .destroy = mock_context_destroy, -}; - -static int mock_request_alloc(struct i915_request *request) -{ - INIT_LIST_HEAD(&request->mock.link); - request->mock.delay = 0; - - return 0; -} - -static int mock_emit_flush(struct i915_request *request, - unsigned int flags) -{ - return 0; -} - -static u32 *mock_emit_breadcrumb(struct i915_request *request, u32 *cs) -{ - return cs; -} - -static void mock_submit_request(struct i915_request *request) -{ - struct mock_engine *engine = - container_of(request->engine, typeof(*engine), base); - unsigned long flags; - - i915_request_submit(request); - - spin_lock_irqsave(&engine->hw_lock, flags); - list_add_tail(&request->mock.link, &engine->hw_queue); - if (list_is_first(&request->mock.link, &engine->hw_queue)) { - if (request->mock.delay) - mod_timer(&engine->hw_delay, - jiffies + request->mock.delay); - else - advance(request); - } - spin_unlock_irqrestore(&engine->hw_lock, flags); -} - -static void mock_reset_prepare(struct intel_engine_cs *engine) -{ -} - -static void mock_reset(struct intel_engine_cs *engine, bool stalled) -{ - GEM_BUG_ON(stalled); -} - -static void mock_reset_finish(struct intel_engine_cs *engine) -{ -} - -static void mock_cancel_requests(struct intel_engine_cs *engine) -{ - struct i915_request *request; - unsigned long flags; - - spin_lock_irqsave(&engine->timeline.lock, flags); - - /* Mark all submitted requests as skipped. */ - list_for_each_entry(request, &engine->timeline.requests, sched.link) { - if (!i915_request_signaled(request)) - dma_fence_set_error(&request->fence, -EIO); - - i915_request_mark_complete(request); - } - - spin_unlock_irqrestore(&engine->timeline.lock, flags); -} - -struct intel_engine_cs *mock_engine(struct drm_i915_private *i915, - const char *name, - int id) -{ - struct mock_engine *engine; - - GEM_BUG_ON(id >= I915_NUM_ENGINES); - - engine = kzalloc(sizeof(*engine) + PAGE_SIZE, GFP_KERNEL); - if (!engine) - return NULL; - - /* minimal engine setup for requests */ - engine->base.i915 = i915; - snprintf(engine->base.name, sizeof(engine->base.name), "%s", name); - engine->base.id = id; - engine->base.mask = BIT(id); - engine->base.status_page.addr = (void *)(engine + 1); - - engine->base.cops = &mock_context_ops; - engine->base.request_alloc = mock_request_alloc; - engine->base.emit_flush = mock_emit_flush; - engine->base.emit_fini_breadcrumb = mock_emit_breadcrumb; - engine->base.submit_request = mock_submit_request; - - engine->base.reset.prepare = mock_reset_prepare; - engine->base.reset.reset = mock_reset; - engine->base.reset.finish = mock_reset_finish; - engine->base.cancel_requests = mock_cancel_requests; - - if (i915_timeline_init(i915, &engine->base.timeline, NULL)) - goto err_free; - i915_timeline_set_subclass(&engine->base.timeline, TIMELINE_ENGINE); - - intel_engine_init_breadcrumbs(&engine->base); - - /* fake hw queue */ - spin_lock_init(&engine->hw_lock); - timer_setup(&engine->hw_delay, hw_delay_complete, 0); - INIT_LIST_HEAD(&engine->hw_queue); - - if (pin_context(i915->kernel_context, &engine->base, - &engine->base.kernel_context)) - goto err_breadcrumbs; - - return &engine->base; - -err_breadcrumbs: - intel_engine_fini_breadcrumbs(&engine->base); - i915_timeline_fini(&engine->base.timeline); -err_free: - kfree(engine); - return NULL; -} - -void mock_engine_flush(struct intel_engine_cs *engine) -{ - struct mock_engine *mock = - container_of(engine, typeof(*mock), base); - struct i915_request *request, *rn; - - del_timer_sync(&mock->hw_delay); - - spin_lock_irq(&mock->hw_lock); - list_for_each_entry_safe(request, rn, &mock->hw_queue, mock.link) - advance(request); - spin_unlock_irq(&mock->hw_lock); -} - -void mock_engine_reset(struct intel_engine_cs *engine) -{ -} - -void mock_engine_free(struct intel_engine_cs *engine) -{ - struct mock_engine *mock = - container_of(engine, typeof(*mock), base); - struct intel_context *ce; - - GEM_BUG_ON(timer_pending(&mock->hw_delay)); - - ce = fetch_and_zero(&engine->last_retired_context); - if (ce) - intel_context_unpin(ce); - - intel_context_unpin(engine->kernel_context); - - intel_engine_fini_breadcrumbs(engine); - i915_timeline_fini(&engine->timeline); - - kfree(engine); -} diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.h b/drivers/gpu/drm/i915/selftests/mock_engine.h deleted file mode 100644 index b9cc3a245f16..000000000000 --- a/drivers/gpu/drm/i915/selftests/mock_engine.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright © 2016 Intel Corporation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - */ - -#ifndef __MOCK_ENGINE_H__ -#define __MOCK_ENGINE_H__ - -#include <linux/list.h> -#include <linux/spinlock.h> -#include <linux/timer.h> - -#include "../intel_ringbuffer.h" - -struct mock_engine { - struct intel_engine_cs base; - - spinlock_t hw_lock; - struct list_head hw_queue; - struct timer_list hw_delay; -}; - -struct intel_engine_cs *mock_engine(struct drm_i915_private *i915, - const char *name, - int id); -void mock_engine_flush(struct intel_engine_cs *engine); -void mock_engine_reset(struct intel_engine_cs *engine); -void mock_engine_free(struct intel_engine_cs *engine); - -#endif /* !__MOCK_ENGINE_H__ */ diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c index 60bbf8b4df40..64bc51400ae7 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c +++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c @@ -25,14 +25,16 @@ #include <linux/pm_domain.h> #include <linux/pm_runtime.h> -#include "mock_engine.h" -#include "mock_context.h" +#include "gt/mock_engine.h" + #include "mock_request.h" #include "mock_gem_device.h" -#include "mock_gem_object.h" #include "mock_gtt.h" #include "mock_uncore.h" +#include "gem/selftests/mock_context.h" +#include "gem/selftests/mock_gem_object.h" + void mock_device_flush(struct drm_i915_private *i915) { struct intel_engine_cs *engine; @@ -40,11 +42,10 @@ void mock_device_flush(struct drm_i915_private *i915) lockdep_assert_held(&i915->drm.struct_mutex); - for_each_engine(engine, i915, id) - mock_engine_flush(engine); - - i915_retire_requests(i915); - GEM_BUG_ON(i915->gt.active_requests); + do { + for_each_engine(engine, i915, id) + mock_engine_flush(engine); + } while (i915_retire_requests(i915)); } static void mock_device_release(struct drm_device *dev) @@ -55,11 +56,9 @@ static void mock_device_release(struct drm_device *dev) mutex_lock(&i915->drm.struct_mutex); mock_device_flush(i915); - i915_gem_contexts_lost(i915); mutex_unlock(&i915->drm.struct_mutex); - drain_delayed_work(&i915->gt.retire_work); - drain_delayed_work(&i915->gt.idle_work); + flush_work(&i915->gem.idle_work); i915_gem_drain_workqueue(i915); mutex_lock(&i915->drm.struct_mutex); @@ -109,10 +108,6 @@ static void mock_retire_work_handler(struct work_struct *work) static void mock_idle_work_handler(struct work_struct *work) { - struct drm_i915_private *i915 = - container_of(work, typeof(*i915), gt.idle_work.work); - - i915->gt.active_engines = 0; } static int pm_domain_resume(struct device *dev) @@ -156,8 +151,6 @@ struct drm_i915_private *mock_gem_device(void) i915 = (struct drm_i915_private *)(pdev + 1); pci_set_drvdata(pdev, i915); - intel_runtime_pm_init_early(i915); - dev_pm_domain_set(&pdev->dev, &pm_domain); pm_runtime_enable(&pdev->dev); pm_runtime_dont_use_autosuspend(&pdev->dev); @@ -172,6 +165,8 @@ struct drm_i915_private *mock_gem_device(void) i915->drm.pdev = pdev; i915->drm.dev_private = i915; + intel_runtime_pm_init_early(&i915->runtime_pm); + /* Using the global GTT may ask questions about KMS users, so prepare */ drm_mode_config_init(&i915->drm); @@ -184,6 +179,8 @@ struct drm_i915_private *mock_gem_device(void) mock_uncore_init(&i915->uncore); i915_gem_init__mm(i915); + intel_gt_pm_init(i915); + atomic_inc(&i915->gt.wakeref.count); /* disable; no hw support */ init_waitqueue_head(&i915->gpu_error.wait_queue); init_waitqueue_head(&i915->gpu_error.reset_queue); @@ -196,8 +193,8 @@ struct drm_i915_private *mock_gem_device(void) mock_init_contexts(i915); - INIT_DELAYED_WORK(&i915->gt.retire_work, mock_retire_work_handler); - INIT_DELAYED_WORK(&i915->gt.idle_work, mock_idle_work_handler); + INIT_DELAYED_WORK(&i915->gem.retire_work, mock_retire_work_handler); + INIT_WORK(&i915->gem.idle_work, mock_idle_work_handler); i915->gt.awake = true; @@ -205,18 +202,23 @@ struct drm_i915_private *mock_gem_device(void) INIT_LIST_HEAD(&i915->gt.active_rings); INIT_LIST_HEAD(&i915->gt.closed_vma); + spin_lock_init(&i915->gt.closed_lock); mutex_lock(&i915->drm.struct_mutex); mock_init_ggtt(i915, &i915->ggtt); mkwrite_device_info(i915)->engine_mask = BIT(0); - i915->kernel_context = mock_context(i915, NULL); - if (!i915->kernel_context) - goto err_unlock; i915->engine[RCS0] = mock_engine(i915, "mock", RCS0); if (!i915->engine[RCS0]) + goto err_unlock; + + i915->kernel_context = mock_context(i915, NULL); + if (!i915->kernel_context) + goto err_engine; + + if (mock_engine_init(i915->engine[RCS0])) goto err_context; mutex_unlock(&i915->drm.struct_mutex); @@ -227,6 +229,8 @@ struct drm_i915_private *mock_gem_device(void) err_context: i915_gem_contexts_fini(i915); +err_engine: + mock_engine_free(i915->engine[RCS0]); err_unlock: mutex_unlock(&i915->drm.struct_mutex); i915_timelines_fini(i915); diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_object.h b/drivers/gpu/drm/i915/selftests/mock_gem_object.h deleted file mode 100644 index 20acdbee7bd0..000000000000 --- a/drivers/gpu/drm/i915/selftests/mock_gem_object.h +++ /dev/null @@ -1,9 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __MOCK_GEM_OBJECT_H__ -#define __MOCK_GEM_OBJECT_H__ - -struct mock_object { - struct drm_i915_gem_object base; -}; - -#endif /* !__MOCK_GEM_OBJECT_H__ */ diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.c b/drivers/gpu/drm/i915/selftests/mock_gtt.c index cd83929fde8e..f625c307a406 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gtt.c +++ b/drivers/gpu/drm/i915/selftests/mock_gtt.c @@ -55,17 +55,14 @@ static void mock_cleanup(struct i915_address_space *vm) { } -struct i915_hw_ppgtt * -mock_ppgtt(struct drm_i915_private *i915, - const char *name) +struct i915_ppgtt *mock_ppgtt(struct drm_i915_private *i915, const char *name) { - struct i915_hw_ppgtt *ppgtt; + struct i915_ppgtt *ppgtt; ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); if (!ppgtt) return NULL; - kref_init(&ppgtt->ref); ppgtt->vm.i915 = i915; ppgtt->vm.total = round_down(U64_MAX, PAGE_SIZE); ppgtt->vm.file = ERR_PTR(-ENODEV); diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.h b/drivers/gpu/drm/i915/selftests/mock_gtt.h index 40d544bde1d5..3387393286de 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gtt.h +++ b/drivers/gpu/drm/i915/selftests/mock_gtt.h @@ -28,8 +28,6 @@ void mock_init_ggtt(struct drm_i915_private *i915, struct i915_ggtt *ggtt); void mock_fini_ggtt(struct i915_ggtt *ggtt); -struct i915_hw_ppgtt * -mock_ppgtt(struct drm_i915_private *i915, - const char *name); +struct i915_ppgtt *mock_ppgtt(struct drm_i915_private *i915, const char *name); #endif /* !__MOCK_GTT_H */ diff --git a/drivers/gpu/drm/i915/selftests/mock_request.c b/drivers/gpu/drm/i915/selftests/mock_request.c index d1a7c9608712..9390fc09984b 100644 --- a/drivers/gpu/drm/i915/selftests/mock_request.c +++ b/drivers/gpu/drm/i915/selftests/mock_request.c @@ -22,7 +22,9 @@ * */ -#include "mock_engine.h" +#include "gem/selftests/igt_gem_utils.h" +#include "gt/mock_engine.h" + #include "mock_request.h" struct i915_request * @@ -33,7 +35,7 @@ mock_request(struct intel_engine_cs *engine, struct i915_request *request; /* NB the i915->requests slab cache is enlarged to fit mock_request */ - request = i915_request_alloc(engine, context); + request = igt_request_alloc(context, engine); if (IS_ERR(request)) return NULL; diff --git a/drivers/gpu/drm/i915/selftests/mock_timeline.c b/drivers/gpu/drm/i915/selftests/mock_timeline.c index e084476469ef..65b52be23d42 100644 --- a/drivers/gpu/drm/i915/selftests/mock_timeline.c +++ b/drivers/gpu/drm/i915/selftests/mock_timeline.c @@ -13,7 +13,6 @@ void mock_timeline_init(struct i915_timeline *timeline, u64 context) timeline->i915 = NULL; timeline->fence_context = context; - spin_lock_init(&timeline->lock); mutex_init(&timeline->mutex); INIT_ACTIVE_REQUEST(&timeline->last_request); diff --git a/drivers/gpu/drm/i915/selftests/scatterlist.c b/drivers/gpu/drm/i915/selftests/scatterlist.c index cd6d2a16071f..d599186d5b71 100644 --- a/drivers/gpu/drm/i915/selftests/scatterlist.c +++ b/drivers/gpu/drm/i915/selftests/scatterlist.c @@ -24,7 +24,8 @@ #include <linux/prime_numbers.h> #include <linux/random.h> -#include "../i915_selftest.h" +#include "i915_selftest.h" +#include "i915_utils.h" #define PFN_BIAS (1 << 10) |