diff options
Diffstat (limited to 'drivers/gpu/drm/nouveau/nvkm/engine/fifo')
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild | 11 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c | 283 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 35 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 656 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h | 178 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c | 183 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv108.c | 37 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c | 220 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c | 361 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 541 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h | 36 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv84.c | 481 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/nvc0.c | 974 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/nve0.c | 1146 | ||||
| -rw-r--r-- | drivers/gpu/drm/nouveau/nvkm/engine/fifo/nve0.h | 18 |
15 files changed, 5160 insertions, 0 deletions
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild new file mode 100644 index 000000000000..f38820faf44f --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild @@ -0,0 +1,11 @@ +nvkm-y += nvkm/engine/fifo/base.o +nvkm-y += nvkm/engine/fifo/nv04.o +nvkm-y += nvkm/engine/fifo/nv10.o +nvkm-y += nvkm/engine/fifo/nv17.o +nvkm-y += nvkm/engine/fifo/nv40.o +nvkm-y += nvkm/engine/fifo/nv50.o +nvkm-y += nvkm/engine/fifo/nv84.o +nvkm-y += nvkm/engine/fifo/nvc0.o +nvkm-y += nvkm/engine/fifo/nve0.o +nvkm-y += nvkm/engine/fifo/gk20a.o +nvkm-y += nvkm/engine/fifo/nv108.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c new file mode 100644 index 000000000000..836c6d32e82e --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -0,0 +1,283 @@ +/* + * Copyright 2012 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Ben Skeggs + */ + +#include <core/client.h> +#include <core/object.h> +#include <core/handle.h> +#include <core/event.h> +#include <nvif/unpack.h> +#include <nvif/class.h> +#include <nvif/event.h> + +#include <engine/dmaobj.h> +#include <engine/fifo.h> + +static int +nouveau_fifo_event_ctor(struct nouveau_object *object, void *data, u32 size, + struct nvkm_notify *notify) +{ + if (size == 0) { + notify->size = 0; + notify->types = 1; + notify->index = 0; + return 0; + } + return -ENOSYS; +} + +static const struct nvkm_event_func +nouveau_fifo_event_func = { + .ctor = nouveau_fifo_event_ctor, +}; + +int +nouveau_fifo_channel_create_(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, + int bar, u32 addr, u32 size, u32 pushbuf, + u64 engmask, int len, void **ptr) +{ + struct nouveau_device *device = nv_device(engine); + struct nouveau_fifo *priv = (void *)engine; + struct nouveau_fifo_chan *chan; + struct nouveau_dmaeng *dmaeng; + unsigned long flags; + int ret; + + /* create base object class */ + ret = nouveau_namedb_create_(parent, engine, oclass, 0, NULL, + engmask, len, ptr); + chan = *ptr; + if (ret) + return ret; + + /* validate dma object representing push buffer */ + chan->pushdma = (void *)nouveau_handle_ref(parent, pushbuf); + if (!chan->pushdma) + return -ENOENT; + + dmaeng = (void *)chan->pushdma->base.engine; + switch (chan->pushdma->base.oclass->handle) { + case NV_DMA_FROM_MEMORY: + case NV_DMA_IN_MEMORY: + break; + default: + return -EINVAL; + } + + ret = dmaeng->bind(chan->pushdma, parent, &chan->pushgpu); + if (ret) + return ret; + + /* find a free fifo channel */ + spin_lock_irqsave(&priv->lock, flags); + for (chan->chid = priv->min; chan->chid < priv->max; chan->chid++) { + if (!priv->channel[chan->chid]) { + priv->channel[chan->chid] = nv_object(chan); + break; + } + } + spin_unlock_irqrestore(&priv->lock, flags); + + if (chan->chid == priv->max) { + nv_error(priv, "no free channels\n"); + return -ENOSPC; + } + + chan->addr = nv_device_resource_start(device, bar) + + addr + size * chan->chid; + chan->size = size; + nvkm_event_send(&priv->cevent, 1, 0, NULL, 0); + return 0; +} + +void +nouveau_fifo_channel_destroy(struct nouveau_fifo_chan *chan) +{ + struct nouveau_fifo *priv = (void *)nv_object(chan)->engine; + unsigned long flags; + + if (chan->user) + iounmap(chan->user); + + spin_lock_irqsave(&priv->lock, flags); + priv->channel[chan->chid] = NULL; + spin_unlock_irqrestore(&priv->lock, flags); + + nouveau_gpuobj_ref(NULL, &chan->pushgpu); + nouveau_object_ref(NULL, (struct nouveau_object **)&chan->pushdma); + nouveau_namedb_destroy(&chan->namedb); +} + +void +_nouveau_fifo_channel_dtor(struct nouveau_object *object) +{ + struct nouveau_fifo_chan *chan = (void *)object; + nouveau_fifo_channel_destroy(chan); +} + +int +_nouveau_fifo_channel_map(struct nouveau_object *object, u64 *addr, u32 *size) +{ + struct nouveau_fifo_chan *chan = (void *)object; + *addr = chan->addr; + *size = chan->size; + return 0; +} + +u32 +_nouveau_fifo_channel_rd32(struct nouveau_object *object, u64 addr) +{ + struct nouveau_fifo_chan *chan = (void *)object; + if (unlikely(!chan->user)) { + chan->user = ioremap(chan->addr, chan->size); + if (WARN_ON_ONCE(chan->user == NULL)) + return 0; + } + return ioread32_native(chan->user + addr); +} + +void +_nouveau_fifo_channel_wr32(struct nouveau_object *object, u64 addr, u32 data) +{ + struct nouveau_fifo_chan *chan = (void *)object; + if (unlikely(!chan->user)) { + chan->user = ioremap(chan->addr, chan->size); + if (WARN_ON_ONCE(chan->user == NULL)) + return; + } + iowrite32_native(data, chan->user + addr); +} + +int +nouveau_fifo_uevent_ctor(struct nouveau_object *object, void *data, u32 size, + struct nvkm_notify *notify) +{ + union { + struct nvif_notify_uevent_req none; + } *req = data; + int ret; + + if (nvif_unvers(req->none)) { + notify->size = sizeof(struct nvif_notify_uevent_rep); + notify->types = 1; + notify->index = 0; + } + + return ret; +} + +void +nouveau_fifo_uevent(struct nouveau_fifo *fifo) +{ + struct nvif_notify_uevent_rep rep = { + }; + nvkm_event_send(&fifo->uevent, 1, 0, &rep, sizeof(rep)); +} + +int +_nouveau_fifo_channel_ntfy(struct nouveau_object *object, u32 type, + struct nvkm_event **event) +{ + struct nouveau_fifo *fifo = (void *)object->engine; + switch (type) { + case G82_CHANNEL_DMA_V0_NTFY_UEVENT: + if (nv_mclass(object) >= G82_CHANNEL_DMA) { + *event = &fifo->uevent; + return 0; + } + break; + default: + break; + } + return -EINVAL; +} + +static int +nouveau_fifo_chid(struct nouveau_fifo *priv, struct nouveau_object *object) +{ + int engidx = nv_hclass(priv) & 0xff; + + while (object && object->parent) { + if ( nv_iclass(object->parent, NV_ENGCTX_CLASS) && + (nv_hclass(object->parent) & 0xff) == engidx) + return nouveau_fifo_chan(object)->chid; + object = object->parent; + } + + return -1; +} + +const char * +nouveau_client_name_for_fifo_chid(struct nouveau_fifo *fifo, u32 chid) +{ + struct nouveau_fifo_chan *chan = NULL; + unsigned long flags; + + spin_lock_irqsave(&fifo->lock, flags); + if (chid >= fifo->min && chid <= fifo->max) + chan = (void *)fifo->channel[chid]; + spin_unlock_irqrestore(&fifo->lock, flags); + + return nouveau_client_name(chan); +} + +void +nouveau_fifo_destroy(struct nouveau_fifo *priv) +{ + kfree(priv->channel); + nvkm_event_fini(&priv->uevent); + nvkm_event_fini(&priv->cevent); + nouveau_engine_destroy(&priv->base); +} + +int +nouveau_fifo_create_(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, + int min, int max, int length, void **pobject) +{ + struct nouveau_fifo *priv; + int ret; + + ret = nouveau_engine_create_(parent, engine, oclass, true, "PFIFO", + "fifo", length, pobject); + priv = *pobject; + if (ret) + return ret; + + priv->min = min; + priv->max = max; + priv->channel = kzalloc(sizeof(*priv->channel) * (max + 1), GFP_KERNEL); + if (!priv->channel) + return -ENOMEM; + + ret = nvkm_event_init(&nouveau_fifo_event_func, 1, 1, &priv->cevent); + if (ret) + return ret; + + priv->chid = nouveau_fifo_chid; + spin_lock_init(&priv->lock); + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c new file mode 100644 index 000000000000..327456eae963 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2014, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "nve0.h" + +struct nouveau_oclass * +gk20a_fifo_oclass = &(struct nve0_fifo_impl) { + .base.handle = NV_ENGINE(FIFO, 0xea), + .base.ofuncs = &(struct nouveau_ofuncs) { + .ctor = nve0_fifo_ctor, + .dtor = nve0_fifo_dtor, + .init = nve0_fifo_init, + .fini = nve0_fifo_fini, + }, + .channels = 128, +}.base; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c new file mode 100644 index 000000000000..6f51ae0daecf --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -0,0 +1,656 @@ +/* + * Copyright 2012 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Ben Skeggs + */ + +#include <core/client.h> +#include <nvif/unpack.h> +#include <nvif/class.h> +#include <core/engctx.h> +#include <core/namedb.h> +#include <core/handle.h> +#include <core/ramht.h> +#include <core/event.h> + +#include <subdev/instmem.h> +#include <subdev/instmem/nv04.h> +#include <subdev/timer.h> +#include <subdev/fb.h> + +#include <engine/fifo.h> + +#include "nv04.h" + +static struct ramfc_desc +nv04_ramfc[] = { + { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, + { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, + { 16, 0, 0x08, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, + { 16, 16, 0x08, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, + { 32, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_STATE }, + { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, + { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_ENGINE }, + { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_PULL1 }, + {} +}; + +/******************************************************************************* + * FIFO channel objects + ******************************************************************************/ + +int +nv04_fifo_object_attach(struct nouveau_object *parent, + struct nouveau_object *object, u32 handle) +{ + struct nv04_fifo_priv *priv = (void *)parent->engine; + struct nv04_fifo_chan *chan = (void *)parent; + u32 context, chid = chan->base.chid; + int ret; + + if (nv_iclass(object, NV_GPUOBJ_CLASS)) + context = nv_gpuobj(object)->addr >> 4; + else + context = 0x00000004; /* just non-zero */ + + switch (nv_engidx(object->engine)) { + case NVDEV_ENGINE_DMAOBJ: + case NVDEV_ENGINE_SW: + context |= 0x00000000; + break; + case NVDEV_ENGINE_GR: + context |= 0x00010000; + break; + case NVDEV_ENGINE_MPEG: + context |= 0x00020000; + break; + default: + return -EINVAL; + } + + context |= 0x80000000; /* valid */ + context |= chid << 24; + + mutex_lock(&nv_subdev(priv)->mutex); + ret = nouveau_ramht_insert(priv->ramht, chid, handle, context); + mutex_unlock(&nv_subdev(priv)->mutex); + return ret; +} + +void +nv04_fifo_object_detach(struct nouveau_object *parent, int cookie) +{ + struct nv04_fifo_priv *priv = (void *)parent->engine; + mutex_lock(&nv_subdev(priv)->mutex); + nouveau_ramht_remove(priv->ramht, cookie); + mutex_unlock(&nv_subdev(priv)->mutex); +} + +int +nv04_fifo_context_attach(struct nouveau_object *parent, + struct nouveau_object *object) +{ + nv_engctx(object)->addr = nouveau_fifo_chan(parent)->chid; + return 0; +} + +static int +nv04_fifo_chan_ctor(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + union { + struct nv03_channel_dma_v0 v0; + } *args = data; + struct nv04_fifo_priv *priv = (void *)engine; + struct nv04_fifo_chan *chan; + int ret; + + nv_ioctl(parent, "create channel dma size %d\n", size); + if (nvif_unpack(args->v0, 0, 0, false)) { + nv_ioctl(parent, "create channel dma vers %d pushbuf %08x " + "offset %016llx\n", args->v0.version, + args->v0.pushbuf, args->v0.offset); + } else + return ret; + + ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0x800000, + 0x10000, args->v0.pushbuf, + (1ULL << NVDEV_ENGINE_DMAOBJ) | + (1ULL << NVDEV_ENGINE_SW) | + (1ULL << NVDEV_ENGINE_GR), &chan); + *pobject = nv_object(chan); + if (ret) + return ret; + + args->v0.chid = chan->base.chid; + + nv_parent(chan)->object_attach = nv04_fifo_object_attach; + nv_parent(chan)->object_detach = nv04_fifo_object_detach; + nv_parent(chan)->context_attach = nv04_fifo_context_attach; + chan->ramfc = chan->base.chid * 32; + + nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->v0.offset); + nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->v0.offset); + nv_wo32(priv->ramfc, chan->ramfc + 0x08, chan->base.pushgpu->addr >> 4); + nv_wo32(priv->ramfc, chan->ramfc + 0x10, + NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES | + NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES | +#ifdef __BIG_ENDIAN + NV_PFIFO_CACHE1_BIG_ENDIAN | +#endif + NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8); + return 0; +} + +void +nv04_fifo_chan_dtor(struct nouveau_object *object) +{ + struct nv04_fifo_priv *priv = (void *)object->engine; + struct nv04_fifo_chan *chan = (void *)object; + struct ramfc_desc *c = priv->ramfc_desc; + + do { + nv_wo32(priv->ramfc, chan->ramfc + c->ctxp, 0x00000000); + } while ((++c)->bits); + + nouveau_fifo_channel_destroy(&chan->base); +} + +int +nv04_fifo_chan_init(struct nouveau_object *object) +{ + struct nv04_fifo_priv *priv = (void *)object->engine; + struct nv04_fifo_chan *chan = (void *)object; + u32 mask = 1 << chan->base.chid; + unsigned long flags; + int ret; + + ret = nouveau_fifo_channel_init(&chan->base); + if (ret) + return ret; + + spin_lock_irqsave(&priv->base.lock, flags); + nv_mask(priv, NV04_PFIFO_MODE, mask, mask); + spin_unlock_irqrestore(&priv->base.lock, flags); + return 0; +} + +int +nv04_fifo_chan_fini(struct nouveau_object *object, bool suspend) +{ + struct nv04_fifo_priv *priv = (void *)object->engine; + struct nv04_fifo_chan *chan = (void *)object; + struct nouveau_gpuobj *fctx = priv->ramfc; + struct ramfc_desc *c; + unsigned long flags; + u32 data = chan->ramfc; + u32 chid; + + /* prevent fifo context switches */ + spin_lock_irqsave(&priv->base.lock, flags); + nv_wr32(priv, NV03_PFIFO_CACHES, 0); + + /* if this channel is active, replace it with a null context */ + chid = nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH1) & priv->base.max; + if (chid == chan->base.chid) { + nv_mask(priv, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0); + nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 0); + nv_mask(priv, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0); + + c = priv->ramfc_desc; + do { + u32 rm = ((1ULL << c->bits) - 1) << c->regs; + u32 cm = ((1ULL << c->bits) - 1) << c->ctxs; + u32 rv = (nv_rd32(priv, c->regp) & rm) >> c->regs; + u32 cv = (nv_ro32(fctx, c->ctxp + data) & ~cm); + nv_wo32(fctx, c->ctxp + data, cv | (rv << c->ctxs)); + } while ((++c)->bits); + + c = priv->ramfc_desc; + do { + nv_wr32(priv, c->regp, 0x00000000); + } while ((++c)->bits); + + nv_wr32(priv, NV03_PFIFO_CACHE1_GET, 0); + nv_wr32(priv, NV03_PFIFO_CACHE1_PUT, 0); + nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH1, priv->base.max); + nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 1); + nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1); + } + + /* restore normal operation, after disabling dma mode */ + nv_mask(priv, NV04_PFIFO_MODE, 1 << chan->base.chid, 0); + nv_wr32(priv, NV03_PFIFO_CACHES, 1); + spin_unlock_irqrestore(&priv->base.lock, flags); + + return nouveau_fifo_channel_fini(&chan->base, suspend); +} + +static struct nouveau_ofuncs +nv04_fifo_ofuncs = { + .ctor = nv04_fifo_chan_ctor, + .dtor = nv04_fifo_chan_dtor, + .init = nv04_fifo_chan_init, + .fini = nv04_fifo_chan_fini, + .map = _nouveau_fifo_channel_map, + .rd32 = _nouveau_fifo_channel_rd32, + .wr32 = _nouveau_fifo_channel_wr32, + .ntfy = _nouveau_fifo_channel_ntfy +}; + +static struct nouveau_oclass +nv04_fifo_sclass[] = { + { NV03_CHANNEL_DMA, &nv04_fifo_ofuncs }, + {} +}; + +/******************************************************************************* + * FIFO context - basically just the instmem reserved for the channel + ******************************************************************************/ + +int +nv04_fifo_context_ctor(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + struct nv04_fifo_base *base; + int ret; + + ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x1000, + 0x1000, NVOBJ_FLAG_HEAP, &base); + *pobject = nv_object(base); + if (ret) + return ret; + + return 0; +} + +static struct nouveau_oclass +nv04_fifo_cclass = { + .handle = NV_ENGCTX(FIFO, 0x04), + .ofuncs = &(struct nouveau_ofuncs) { + .ctor = nv04_fifo_context_ctor, + .dtor = _nouveau_fifo_context_dtor, + .init = _nouveau_fifo_context_init, + .fini = _nouveau_fifo_context_fini, + .rd32 = _nouveau_fifo_context_rd32, + .wr32 = _nouveau_fifo_context_wr32, + }, +}; + +/******************************************************************************* + * PFIFO engine + ******************************************************************************/ + +void +nv04_fifo_pause(struct nouveau_fifo *pfifo, unsigned long *pflags) +__acquires(priv->base.lock) +{ + struct nv04_fifo_priv *priv = (void *)pfifo; + unsigned long flags; + + spin_lock_irqsave(&priv->base.lock, flags); + *pflags = flags; + + nv_wr32(priv, NV03_PFIFO_CACHES, 0x00000000); + nv_mask(priv, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000000); + + /* in some cases the puller may be left in an inconsistent state + * if you try to stop it while it's busy translating handles. + * sometimes you get a CACHE_ERROR, sometimes it just fails + * silently; sending incorrect instance offsets to PGRAPH after + * it's started up again. + * + * to avoid this, we invalidate the most recently calculated + * instance. + */ + if (!nv_wait(priv, NV04_PFIFO_CACHE1_PULL0, + NV04_PFIFO_CACHE1_PULL0_HASH_BUSY, 0x00000000)) + nv_warn(priv, "timeout idling puller\n"); + + if (nv_rd32(priv, NV04_PFIFO_CACHE1_PULL0) & + NV04_PFIFO_CACHE1_PULL0_HASH_FAILED) + nv_wr32(priv, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR); + + nv_wr32(priv, NV04_PFIFO_CACHE1_HASH, 0x00000000); +} + +void +nv04_fifo_start(struct nouveau_fifo *pfifo, unsigned long *pflags) +__releases(priv->base.lock) +{ + struct nv04_fifo_priv *priv = (void *)pfifo; + unsigned long flags = *pflags; + + nv_mask(priv, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000001); + nv_wr32(priv, NV03_PFIFO_CACHES, 0x00000001); + + spin_unlock_irqrestore(&priv->base.lock, flags); +} + +static const char * +nv_dma_state_err(u32 state) +{ + static const char * const desc[] = { + "NONE", "CALL_SUBR_ACTIVE", "INVALID_MTHD", "RET_SUBR_INACTIVE", + "INVALID_CMD", "IB_EMPTY"/* NV50+ */, "MEM_FAULT", "UNK" + }; + return desc[(state >> 29) & 0x7]; +} + +static bool +nv04_fifo_swmthd(struct nv04_fifo_priv *priv, u32 chid, u32 addr, u32 data) +{ + struct nv04_fifo_chan *chan = NULL; + struct nouveau_handle *bind; + const int subc = (addr >> 13) & 0x7; + const int mthd = addr & 0x1ffc; + bool handled = false; + unsigned long flags; + u32 engine; + + spin_lock_irqsave(&priv->base.lock, flags); + if (likely(chid >= priv->base.min && chid <= priv->base.max)) + chan = (void *)priv->base.channel[chid]; + if (unlikely(!chan)) + goto out; + + switch (mthd) { + case 0x0000: + bind = nouveau_namedb_get(nv_namedb(chan), data); + if (unlikely(!bind)) + break; + + if (nv_engidx(bind->object->engine) == NVDEV_ENGINE_SW) { + engine = 0x0000000f << (subc * 4); + chan->subc[subc] = data; + handled = true; + + nv_mask(priv, NV04_PFIFO_CACHE1_ENGINE, engine, 0); + } + + nouveau_namedb_put(bind); + break; + default: + engine = nv_rd32(priv, NV04_PFIFO_CACHE1_ENGINE); + if (unlikely(((engine >> (subc * 4)) & 0xf) != 0)) + break; + + bind = nouveau_namedb_get(nv_namedb(chan), chan->subc[subc]); + if (likely(bind)) { + if (!nv_call(bind->object, mthd, data)) + handled = true; + nouveau_namedb_put(bind); + } + break; + } + +out: + spin_unlock_irqrestore(&priv->base.lock, flags); + return handled; +} + +static void +nv04_fifo_cache_error(struct nouveau_device *device, + struct nv04_fifo_priv *priv, u32 chid, u32 get) +{ + u32 mthd, data; + int ptr; + + /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before wrapping on my + * G80 chips, but CACHE1 isn't big enough for this much data.. Tests + * show that it wraps around to the start at GET=0x800.. No clue as to + * why.. + */ + ptr = (get & 0x7ff) >> 2; + + if (device->card_type < NV_40) { + mthd = nv_rd32(priv, NV04_PFIFO_CACHE1_METHOD(ptr)); + data = nv_rd32(priv, NV04_PFIFO_CACHE1_DATA(ptr)); + } else { + mthd = nv_rd32(priv, NV40_PFIFO_CACHE1_METHOD(ptr)); + data = nv_rd32(priv, NV40_PFIFO_CACHE1_DATA(ptr)); + } + + if (!nv04_fifo_swmthd(priv, chid, mthd, data)) { + const char *client_name = + nouveau_client_name_for_fifo_chid(&priv->base, chid); + nv_error(priv, + "CACHE_ERROR - ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n", + chid, client_name, (mthd >> 13) & 7, mthd & 0x1ffc, + data); + } + + nv_wr32(priv, NV04_PFIFO_CACHE1_DMA_PUSH, 0); + nv_wr32(priv, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR); + + nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, + nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH0) & ~1); + nv_wr32(priv, NV03_PFIFO_CACHE1_GET, get + 4); + nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, + nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH0) | 1); + nv_wr32(priv, NV04_PFIFO_CACHE1_HASH, 0); + + nv_wr32(priv, NV04_PFIFO_CACHE1_DMA_PUSH, + nv_rd32(priv, NV04_PFIFO_CACHE1_DMA_PUSH) | 1); + nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1); +} + +static void +nv04_fifo_dma_pusher(struct nouveau_device *device, struct nv04_fifo_priv *priv, + u32 chid) +{ + const char *client_name; + u32 dma_get = nv_rd32(priv, 0x003244); + u32 dma_put = nv_rd32(priv, 0x003240); + u32 push = nv_rd32(priv, 0x003220); + u32 state = nv_rd32(priv, 0x003228); + + client_name = nouveau_client_name_for_fifo_chid(&priv->base, chid); + + if (device->card_type == NV_50) { + u32 ho_get = nv_rd32(priv, 0x003328); + u32 ho_put = nv_rd32(priv, 0x003320); + u32 ib_get = nv_rd32(priv, 0x003334); + u32 ib_put = nv_rd32(priv, 0x003330); + + nv_error(priv, + "DMA_PUSHER - ch %d [%s] get 0x%02x%08x put 0x%02x%08x ib_get 0x%08x ib_put 0x%08x state 0x%08x (err: %s) push 0x%08x\n", + chid, client_name, ho_get, dma_get, ho_put, dma_put, + ib_get, ib_put, state, nv_dma_state_err(state), push); + + /* METHOD_COUNT, in DMA_STATE on earlier chipsets */ + nv_wr32(priv, 0x003364, 0x00000000); + if (dma_get != dma_put || ho_get != ho_put) { + nv_wr32(priv, 0x003244, dma_put); + nv_wr32(priv, 0x003328, ho_put); + } else + if (ib_get != ib_put) + nv_wr32(priv, 0x003334, ib_put); + } else { + nv_error(priv, + "DMA_PUSHER - ch %d [%s] get 0x%08x put 0x%08x state 0x%08x (err: %s) push 0x%08x\n", + chid, client_name, dma_get, dma_put, state, + nv_dma_state_err(state), push); + + if (dma_get != dma_put) + nv_wr32(priv, 0x003244, dma_put); + } + + nv_wr32(priv, 0x003228, 0x00000000); + nv_wr32(priv, 0x003220, 0x00000001); + nv_wr32(priv, 0x002100, NV_PFIFO_INTR_DMA_PUSHER); +} + +void +nv04_fifo_intr(struct nouveau_subdev *subdev) +{ + struct nouveau_device *device = nv_device(subdev); + struct nv04_fifo_priv *priv = (void *)subdev; + uint32_t status, reassign; + int cnt = 0; + + reassign = nv_rd32(priv, NV03_PFIFO_CACHES) & 1; + while ((status = nv_rd32(priv, NV03_PFIFO_INTR_0)) && (cnt++ < 100)) { + uint32_t chid, get; + + nv_wr32(priv, NV03_PFIFO_CACHES, 0); + + chid = nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH1) & priv->base.max; + get = nv_rd32(priv, NV03_PFIFO_CACHE1_GET); + + if (status & NV_PFIFO_INTR_CACHE_ERROR) { + nv04_fifo_cache_error(device, priv, chid, get); + status &= ~NV_PFIFO_INTR_CACHE_ERROR; + } + + if (status & NV_PFIFO_INTR_DMA_PUSHER) { + nv04_fifo_dma_pusher(device, priv, chid); + status &= ~NV_PFIFO_INTR_DMA_PUSHER; + } + + if (status & NV_PFIFO_INTR_SEMAPHORE) { + uint32_t sem; + + status &= ~NV_PFIFO_INTR_SEMAPHORE; + nv_wr32(priv, NV03_PFIFO_INTR_0, + NV_PFIFO_INTR_SEMAPHORE); + + sem = nv_rd32(priv, NV10_PFIFO_CACHE1_SEMAPHORE); + nv_wr32(priv, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1); + + nv_wr32(priv, NV03_PFIFO_CACHE1_GET, get + 4); + nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1); + } + + if (device->card_type == NV_50) { + if (status & 0x00000010) { + status &= ~0x00000010; + nv_wr32(priv, 0x002100, 0x00000010); + } + + if (status & 0x40000000) { + nv_wr32(priv, 0x002100, 0x40000000); + nouveau_fifo_uevent(&priv->base); + status &= ~0x40000000; + } + } + + if (status) { + nv_warn(priv, "unknown intr 0x%08x, ch %d\n", + status, chid); + nv_wr32(priv, NV03_PFIFO_INTR_0, status); + status = 0; + } + + nv_wr32(priv, NV03_PFIFO_CACHES, reassign); + } + + if (status) { + nv_error(priv, "still angry after %d spins, halt\n", cnt); + nv_wr32(priv, 0x002140, 0); + nv_wr32(priv, 0x000140, 0); + } + + nv_wr32(priv, 0x000100, 0x00000100); +} + +static int +nv04_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + struct nv04_instmem_priv *imem = nv04_instmem(parent); + struct nv04_fifo_priv *priv; + int ret; + + ret = nouveau_fifo_create(parent, engine, oclass, 0, 15, &priv); + *pobject = nv_object(priv); + if (ret) + return ret; + + nouveau_ramht_ref(imem->ramht, &priv->ramht); + nouveau_gpuobj_ref(imem->ramro, &priv->ramro); + nouveau_gpuobj_ref(imem->ramfc, &priv->ramfc); + + nv_subdev(priv)->unit = 0x00000100; + nv_subdev(priv)->intr = nv04_fifo_intr; + nv_engine(priv)->cclass = &nv04_fifo_cclass; + nv_engine(priv)->sclass = nv04_fifo_sclass; + priv->base.pause = nv04_fifo_pause; + priv->base.start = nv04_fifo_start; + priv->ramfc_desc = nv04_ramfc; + return 0; +} + +void +nv04_fifo_dtor(struct nouveau_object *object) +{ + struct nv04_fifo_priv *priv = (void *)object; + nouveau_gpuobj_ref(NULL, &priv->ramfc); + nouveau_gpuobj_ref(NULL, &priv->ramro); + nouveau_ramht_ref(NULL, &priv->ramht); + nouveau_fifo_destroy(&priv->base); +} + +int +nv04_fifo_init(struct nouveau_object *object) +{ + struct nv04_fifo_priv *priv = (void *)object; + int ret; + + ret = nouveau_fifo_init(&priv->base); + if (ret) + return ret; + + nv_wr32(priv, NV04_PFIFO_DELAY_0, 0x000000ff); + nv_wr32(priv, NV04_PFIFO_DMA_TIMESLICE, 0x0101ffff); + + nv_wr32(priv, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ | + ((priv->ramht->bits - 9) << 16) | + (priv->ramht->gpuobj.addr >> 8)); + nv_wr32(priv, NV03_PFIFO_RAMRO, priv->ramro->addr >> 8); + nv_wr32(priv, NV03_PFIFO_RAMFC, priv->ramfc->addr >> 8); + + nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH1, priv->base.max); + + nv_wr32(priv, NV03_PFIFO_INTR_0, 0xffffffff); + nv_wr32(priv, NV03_PFIFO_INTR_EN_0, 0xffffffff); + + nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 1); + nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1); + nv_wr32(priv, NV03_PFIFO_CACHES, 1); + return 0; +} + +struct nouveau_oclass * +nv04_fifo_oclass = &(struct nouveau_oclass) { + .handle = NV_ENGINE(FIFO, 0x04), + .ofuncs = &(struct nouveau_ofuncs) { + .ctor = nv04_fifo_ctor, + .dtor = nv04_fifo_dtor, + .init = nv04_fifo_init, + .fini = _nouveau_fifo_fini, + }, +}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h new file mode 100644 index 000000000000..496a4b4fdfaf --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h @@ -0,0 +1,178 @@ +#ifndef __NV04_FIFO_H__ +#define __NV04_FIFO_H__ + +#include <engine/fifo.h> + +#define NV04_PFIFO_DELAY_0 0x00002040 +#define NV04_PFIFO_DMA_TIMESLICE 0x00002044 +#define NV04_PFIFO_NEXT_CHANNEL 0x00002050 +#define NV03_PFIFO_INTR_0 0x00002100 +#define NV03_PFIFO_INTR_EN_0 0x00002140 +# define NV_PFIFO_INTR_CACHE_ERROR (1<<0) +# define NV_PFIFO_INTR_RUNOUT (1<<4) +# define NV_PFIFO_INTR_RUNOUT_OVERFLOW (1<<8) +# define NV_PFIFO_INTR_DMA_PUSHER (1<<12) +# define NV_PFIFO_INTR_DMA_PT (1<<16) +# define NV_PFIFO_INTR_SEMAPHORE (1<<20) +# define NV_PFIFO_INTR_ACQUIRE_TIMEOUT (1<<24) +#define NV03_PFIFO_RAMHT 0x00002210 +#define NV03_PFIFO_RAMFC 0x00002214 +#define NV03_PFIFO_RAMRO 0x00002218 +#define NV40_PFIFO_RAMFC 0x00002220 +#define NV03_PFIFO_CACHES 0x00002500 +#define NV04_PFIFO_MODE 0x00002504 +#define NV04_PFIFO_DMA 0x00002508 +#define NV04_PFIFO_SIZE 0x0000250c +#define NV50_PFIFO_CTX_TABLE(c) (0x2600+(c)*4) +#define NV50_PFIFO_CTX_TABLE__SIZE 128 +#define NV50_PFIFO_CTX_TABLE_CHANNEL_ENABLED (1<<31) +#define NV50_PFIFO_CTX_TABLE_UNK30_BAD (1<<30) +#define NV50_PFIFO_CTX_TABLE_INSTANCE_MASK_G80 0x0FFFFFFF +#define NV50_PFIFO_CTX_TABLE_INSTANCE_MASK_G84 0x00FFFFFF +#define NV03_PFIFO_CACHE0_PUSH0 0x00003000 +#define NV03_PFIFO_CACHE0_PULL0 0x00003040 +#define NV04_PFIFO_CACHE0_PULL0 0x00003050 +#define NV04_PFIFO_CACHE0_PULL1 0x00003054 +#define NV03_PFIFO_CACHE1_PUSH0 0x00003200 +#define NV03_PFIFO_CACHE1_PUSH1 0x00003204 +#define NV03_PFIFO_CACHE1_PUSH1_DMA (1<<8) +#define NV40_PFIFO_CACHE1_PUSH1_DMA (1<<16) +#define NV03_PFIFO_CACHE1_PUSH1_CHID_MASK 0x0000000f +#define NV10_PFIFO_CACHE1_PUSH1_CHID_MASK 0x0000001f +#define NV50_PFIFO_CACHE1_PUSH1_CHID_MASK 0x0000007f +#define NV03_PFIFO_CACHE1_PUT 0x00003210 +#define NV04_PFIFO_CACHE1_DMA_PUSH 0x00003220 +#define NV04_PFIFO_CACHE1_DMA_FETCH 0x00003224 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_8_BYTES 0x00000000 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_16_BYTES 0x00000008 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_24_BYTES 0x00000010 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_32_BYTES 0x00000018 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_40_BYTES 0x00000020 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_48_BYTES 0x00000028 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_56_BYTES 0x00000030 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_64_BYTES 0x00000038 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_72_BYTES 0x00000040 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_80_BYTES 0x00000048 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_88_BYTES 0x00000050 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_96_BYTES 0x00000058 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_104_BYTES 0x00000060 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_112_BYTES 0x00000068 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_120_BYTES 0x00000070 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES 0x00000078 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_136_BYTES 0x00000080 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_144_BYTES 0x00000088 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_152_BYTES 0x00000090 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_160_BYTES 0x00000098 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_168_BYTES 0x000000A0 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_176_BYTES 0x000000A8 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_184_BYTES 0x000000B0 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_192_BYTES 0x000000B8 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_200_BYTES 0x000000C0 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_208_BYTES 0x000000C8 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_216_BYTES 0x000000D0 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_224_BYTES 0x000000D8 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_232_BYTES 0x000000E0 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_240_BYTES 0x000000E8 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_248_BYTES 0x000000F0 +# define NV_PFIFO_CACHE1_DMA_FETCH_TRIG_256_BYTES 0x000000F8 +# define NV_PFIFO_CACHE1_DMA_FETCH_SIZE 0x0000E000 +# define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_32_BYTES 0x00000000 +# define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_64_BYTES 0x00002000 +# define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_96_BYTES 0x00004000 +# define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES 0x00006000 +# define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_160_BYTES 0x00008000 +# define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_192_BYTES 0x0000A000 +# define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_224_BYTES 0x0000C000 +# define NV_PFIFO_CACHE1_DMA_FETCH_SIZE_256_BYTES 0x0000E000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS 0x001F0000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_0 0x00000000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_1 0x00010000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_2 0x00020000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_3 0x00030000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_4 0x00040000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_5 0x00050000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_6 0x00060000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_7 0x00070000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8 0x00080000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_9 0x00090000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_10 0x000A0000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_11 0x000B0000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_12 0x000C0000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_13 0x000D0000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_14 0x000E0000 +# define NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_15 0x000F0000 +# define NV_PFIFO_CACHE1_ENDIAN 0x80000000 +# define NV_PFIFO_CACHE1_LITTLE_ENDIAN 0x7FFFFFFF +# define NV_PFIFO_CACHE1_BIG_ENDIAN 0x80000000 +#define NV04_PFIFO_CACHE1_DMA_STATE 0x00003228 +#define NV04_PFIFO_CACHE1_DMA_INSTANCE 0x0000322c +#define NV04_PFIFO_CACHE1_DMA_CTL 0x00003230 +#define NV04_PFIFO_CACHE1_DMA_PUT 0x00003240 +#define NV04_PFIFO_CACHE1_DMA_GET 0x00003244 +#define NV10_PFIFO_CACHE1_REF_CNT 0x00003248 +#define NV10_PFIFO_CACHE1_DMA_SUBROUTINE 0x0000324C +#define NV03_PFIFO_CACHE1_PULL0 0x00003240 +#define NV04_PFIFO_CACHE1_PULL0 0x00003250 +# define NV04_PFIFO_CACHE1_PULL0_HASH_FAILED 0x00000010 +# define NV04_PFIFO_CACHE1_PULL0_HASH_BUSY 0x00001000 +#define NV03_PFIFO_CACHE1_PULL1 0x00003250 +#define NV04_PFIFO_CACHE1_PULL1 0x00003254 +#define NV04_PFIFO_CACHE1_HASH 0x00003258 +#define NV10_PFIFO_CACHE1_ACQUIRE_TIMEOUT 0x00003260 +#define NV10_PFIFO_CACHE1_ACQUIRE_TIMESTAMP 0x00003264 +#define NV10_PFIFO_CACHE1_ACQUIRE_VALUE 0x00003268 +#define NV10_PFIFO_CACHE1_SEMAPHORE 0x0000326C +#define NV03_PFIFO_CACHE1_GET 0x00003270 +#define NV04_PFIFO_CACHE1_ENGINE 0x00003280 +#define NV04_PFIFO_CACHE1_DMA_DCOUNT 0x000032A0 +#define NV40_PFIFO_GRCTX_INSTANCE 0x000032E0 +#define NV40_PFIFO_UNK32E4 0x000032E4 +#define NV04_PFIFO_CACHE1_METHOD(i) (0x00003800+(i*8)) +#define NV04_PFIFO_CACHE1_DATA(i) (0x00003804+(i*8)) +#define NV40_PFIFO_CACHE1_METHOD(i) (0x00090000+(i*8)) +#define NV40_PFIFO_CACHE1_DATA(i) (0x00090004+(i*8)) + +struct ramfc_desc { + unsigned bits:6; + unsigned ctxs:5; + unsigned ctxp:8; + unsigned regs:5; + unsigned regp; +}; + +struct nv04_fifo_priv { + struct nouveau_fifo base; + struct ramfc_desc *ramfc_desc; + struct nouveau_ramht *ramht; + struct nouveau_gpuobj *ramro; + struct nouveau_gpuobj *ramfc; +}; + +struct nv04_fifo_base { + struct nouveau_fifo_base base; +}; + +struct nv04_fifo_chan { + struct nouveau_fifo_chan base; + u32 subc[8]; + u32 ramfc; +}; + +int nv04_fifo_object_attach(struct nouveau_object *, + struct nouveau_object *, u32); +void nv04_fifo_object_detach(struct nouveau_object *, int); + +void nv04_fifo_chan_dtor(struct nouveau_object *); +int nv04_fifo_chan_init(struct nouveau_object *); +int nv04_fifo_chan_fini(struct nouveau_object *, bool suspend); + +int nv04_fifo_context_ctor(struct nouveau_object *, struct nouveau_object *, + struct nouveau_oclass *, void *, u32, + struct nouveau_object **); + +void nv04_fifo_dtor(struct nouveau_object *); +int nv04_fifo_init(struct nouveau_object *); +void nv04_fifo_pause(struct nouveau_fifo *, unsigned long *); +void nv04_fifo_start(struct nouveau_fifo *, unsigned long *); + +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c new file mode 100644 index 000000000000..2a32add51c81 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c @@ -0,0 +1,183 @@ +/* + * Copyright 2012 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Ben Skeggs + */ + +#include <core/client.h> +#include <nvif/unpack.h> +#include <nvif/class.h> +#include <core/engctx.h> +#include <core/ramht.h> + +#include <subdev/instmem.h> +#include <subdev/instmem/nv04.h> +#include <subdev/fb.h> + +#include <engine/fifo.h> + +#include "nv04.h" + +static struct ramfc_desc +nv10_ramfc[] = { + { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, + { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, + { 32, 0, 0x08, 0, NV10_PFIFO_CACHE1_REF_CNT }, + { 16, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, + { 16, 16, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, + { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_STATE }, + { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, + { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_ENGINE }, + { 32, 0, 0x1c, 0, NV04_PFIFO_CACHE1_PULL1 }, + {} +}; + +/******************************************************************************* + * FIFO channel objects + ******************************************************************************/ + +static int +nv10_fifo_chan_ctor(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + union { + struct nv03_channel_dma_v0 v0; + } *args = data; + struct nv04_fifo_priv *priv = (void *)engine; + struct nv04_fifo_chan *chan; + int ret; + + nv_ioctl(parent, "create channel dma size %d\n", size); + if (nvif_unpack(args->v0, 0, 0, false)) { + nv_ioctl(parent, "create channel dma vers %d pushbuf %08x " + "offset %016llx\n", args->v0.version, + args->v0.pushbuf, args->v0.offset); + } else + return ret; + + ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0x800000, + 0x10000, args->v0.pushbuf, + (1ULL << NVDEV_ENGINE_DMAOBJ) | + (1ULL << NVDEV_ENGINE_SW) | + (1ULL << NVDEV_ENGINE_GR), &chan); + *pobject = nv_object(chan); + if (ret) + return ret; + + args->v0.chid = chan->base.chid; + + nv_parent(chan)->object_attach = nv04_fifo_object_attach; + nv_parent(chan)->object_detach = nv04_fifo_object_detach; + nv_parent(chan)->context_attach = nv04_fifo_context_attach; + chan->ramfc = chan->base.chid * 32; + + nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->v0.offset); + nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->v0.offset); + nv_wo32(priv->ramfc, chan->ramfc + 0x0c, chan->base.pushgpu->addr >> 4); + nv_wo32(priv->ramfc, chan->ramfc + 0x14, + NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES | + NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES | +#ifdef __BIG_ENDIAN + NV_PFIFO_CACHE1_BIG_ENDIAN | +#endif + NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8); + return 0; +} + +static struct nouveau_ofuncs +nv10_fifo_ofuncs = { + .ctor = nv10_fifo_chan_ctor, + .dtor = nv04_fifo_chan_dtor, + .init = nv04_fifo_chan_init, + .fini = nv04_fifo_chan_fini, + .map = _nouveau_fifo_channel_map, + .rd32 = _nouveau_fifo_channel_rd32, + .wr32 = _nouveau_fifo_channel_wr32, + .ntfy = _nouveau_fifo_channel_ntfy +}; + +static struct nouveau_oclass +nv10_fifo_sclass[] = { + { NV10_CHANNEL_DMA, &nv10_fifo_ofuncs }, + {} +}; + +/******************************************************************************* + * FIFO context - basically just the instmem reserved for the channel + ******************************************************************************/ + +static struct nouveau_oclass +nv10_fifo_cclass = { + .handle = NV_ENGCTX(FIFO, 0x10), + .ofuncs = &(struct nouveau_ofuncs) { + .ctor = nv04_fifo_context_ctor, + .dtor = _nouveau_fifo_context_dtor, + .init = _nouveau_fifo_context_init, + .fini = _nouveau_fifo_context_fini, + .rd32 = _nouveau_fifo_context_rd32, + .wr32 = _nouveau_fifo_context_wr32, + }, +}; + +/******************************************************************************* + * PFIFO engine + ******************************************************************************/ + +static int +nv10_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + struct nv04_instmem_priv *imem = nv04_instmem(parent); + struct nv04_fifo_priv *priv; + int ret; + + ret = nouveau_fifo_create(parent, engine, oclass, 0, 31, &priv); + *pobject = nv_object(priv); + if (ret) + return ret; + + nouveau_ramht_ref(imem->ramht, &priv->ramht); + nouveau_gpuobj_ref(imem->ramro, &priv->ramro); + nouveau_gpuobj_ref(imem->ramfc, &priv->ramfc); + + nv_subdev(priv)->unit = 0x00000100; + nv_subdev(priv)->intr = nv04_fifo_intr; + nv_engine(priv)->cclass = &nv10_fifo_cclass; + nv_engine(priv)->sclass = nv10_fifo_sclass; + priv->base.pause = nv04_fifo_pause; + priv->base.start = nv04_fifo_start; + priv->ramfc_desc = nv10_ramfc; + return 0; +} + +struct nouveau_oclass * +nv10_fifo_oclass = &(struct nouveau_oclass) { + .handle = NV_ENGINE(FIFO, 0x10), + .ofuncs = &(struct nouveau_ofuncs) { + .ctor = nv10_fifo_ctor, + .dtor = nv04_fifo_dtor, + .init = nv04_fifo_init, + .fini = _nouveau_fifo_fini, + }, +}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv108.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv108.c new file mode 100644 index 000000000000..09362a51ba57 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv108.c @@ -0,0 +1,37 @@ +/* + * Copyright 2013 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Ben Skeggs + */ + +#include "nve0.h" + +struct nouveau_oclass * +nv108_fifo_oclass = &(struct nve0_fifo_impl) { + .base.handle = NV_ENGINE(FIFO, 0x08), + .base.ofuncs = &(struct nouveau_ofuncs) { + .ctor = nve0_fifo_ctor, + .dtor = nve0_fifo_dtor, + .init = nve0_fifo_init, + .fini = _nouveau_fifo_fini, + }, + .channels = 1024, +}.base; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c new file mode 100644 index 000000000000..01fbb11af00d --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c @@ -0,0 +1,220 @@ +/* + * Copyright 2012 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Ben Skeggs + */ + +#include <core/client.h> +#include <nvif/unpack.h> +#include <nvif/class.h> +#include <core/engctx.h> +#include <core/ramht.h> + +#include <subdev/instmem.h> +#include <subdev/instmem/nv04.h> +#include <subdev/fb.h> + +#include <engine/fifo.h> + +#include "nv04.h" + +static struct ramfc_desc +nv17_ramfc[] = { + { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, + { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, + { 32, 0, 0x08, 0, NV10_PFIFO_CACHE1_REF_CNT }, + { 16, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, + { 16, 16, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, + { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_STATE }, + { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, + { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_ENGINE }, + { 32, 0, 0x1c, 0, NV04_PFIFO_CACHE1_PULL1 }, + { 32, 0, 0x20, 0, NV10_PFIFO_CACHE1_ACQUIRE_VALUE }, + { 32, 0, 0x24, 0, NV10_PFIFO_CACHE1_ACQUIRE_TIMESTAMP }, + { 32, 0, 0x28, 0, NV10_PFIFO_CACHE1_ACQUIRE_TIMEOUT }, + { 32, 0, 0x2c, 0, NV10_PFIFO_CACHE1_SEMAPHORE }, + { 32, 0, 0x30, 0, NV10_PFIFO_CACHE1_DMA_SUBROUTINE }, + {} +}; + +/******************************************************************************* + * FIFO channel objects + ******************************************************************************/ + +static int +nv17_fifo_chan_ctor(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + union { + struct nv03_channel_dma_v0 v0; + } *args = data; + struct nv04_fifo_priv *priv = (void *)engine; + struct nv04_fifo_chan *chan; + int ret; + + nv_ioctl(parent, "create channel dma size %d\n", size); + if (nvif_unpack(args->v0, 0, 0, false)) { + nv_ioctl(parent, "create channel dma vers %d pushbuf %08x " + "offset %016llx\n", args->v0.version, + args->v0.pushbuf, args->v0.offset); + } else + return ret; + + ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0x800000, + 0x10000, args->v0.pushbuf, + (1ULL << NVDEV_ENGINE_DMAOBJ) | + (1ULL << NVDEV_ENGINE_SW) | + (1ULL << NVDEV_ENGINE_GR) | + (1ULL << NVDEV_ENGINE_MPEG), /* NV31- */ + &chan); + *pobject = nv_object(chan); + if (ret) + return ret; + + args->v0.chid = chan->base.chid; + + nv_parent(chan)->object_attach = nv04_fifo_object_attach; + nv_parent(chan)->object_detach = nv04_fifo_object_detach; + nv_parent(chan)->context_attach = nv04_fifo_context_attach; + chan->ramfc = chan->base.chid * 64; + + nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->v0.offset); + nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->v0.offset); + nv_wo32(priv->ramfc, chan->ramfc + 0x0c, chan->base.pushgpu->addr >> 4); + nv_wo32(priv->ramfc, chan->ramfc + 0x14, + NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES | + NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES | +#ifdef __BIG_ENDIAN + NV_PFIFO_CACHE1_BIG_ENDIAN | +#endif + NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8); + return 0; +} + +static struct nouveau_ofuncs +nv17_fifo_ofuncs = { + .ctor = nv17_fifo_chan_ctor, + .dtor = nv04_fifo_chan_dtor, + .init = nv04_fifo_chan_init, + .fini = nv04_fifo_chan_fini, + .map = _nouveau_fifo_channel_map, + .rd32 = _nouveau_fifo_channel_rd32, + .wr32 = _nouveau_fifo_channel_wr32, + .ntfy = _nouveau_fifo_channel_ntfy +}; + +static struct nouveau_oclass +nv17_fifo_sclass[] = { + { NV17_CHANNEL_DMA, &nv17_fifo_ofuncs }, + {} +}; + +/******************************************************************************* + * FIFO context - basically just the instmem reserved for the channel + ******************************************************************************/ + +static struct nouveau_oclass +nv17_fifo_cclass = { + .handle = NV_ENGCTX(FIFO, 0x17), + .ofuncs = &(struct nouveau_ofuncs) { + .ctor = nv04_fifo_context_ctor, + .dtor = _nouveau_fifo_context_dtor, + .init = _nouveau_fifo_context_init, + .fini = _nouveau_fifo_context_fini, + .rd32 = _nouveau_fifo_context_rd32, + .wr32 = _nouveau_fifo_context_wr32, + }, +}; + +/******************************************************************************* + * PFIFO engine + ******************************************************************************/ + +static int +nv17_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + struct nv04_instmem_priv *imem = nv04_instmem(parent); + struct nv04_fifo_priv *priv; + int ret; + + ret = nouveau_fifo_create(parent, engine, oclass, 0, 31, &priv); + *pobject = nv_object(priv); + if (ret) + return ret; + + nouveau_ramht_ref(imem->ramht, &priv->ramht); + nouveau_gpuobj_ref(imem->ramro, &priv->ramro); + nouveau_gpuobj_ref(imem->ramfc, &priv->ramfc); + + nv_subdev(priv)->unit = 0x00000100; + nv_subdev(priv)->intr = nv04_fifo_intr; + nv_engine(priv)->cclass = &nv17_fifo_cclass; + nv_engine(priv)->sclass = nv17_fifo_sclass; + priv->base.pause = nv04_fifo_pause; + priv->base.start = nv04_fifo_start; + priv->ramfc_desc = nv17_ramfc; + return 0; +} + +static int +nv17_fifo_init(struct nouveau_object *object) +{ + struct nv04_fifo_priv *priv = (void *)object; + int ret; + + ret = nouveau_fifo_init(&priv->base); + if (ret) + return ret; + + nv_wr32(priv, NV04_PFIFO_DELAY_0, 0x000000ff); + nv_wr32(priv, NV04_PFIFO_DMA_TIMESLICE, 0x0101ffff); + + nv_wr32(priv, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ | + ((priv->ramht->bits - 9) << 16) | + (priv->ramht->gpuobj.addr >> 8)); + nv_wr32(priv, NV03_PFIFO_RAMRO, priv->ramro->addr >> 8); + nv_wr32(priv, NV03_PFIFO_RAMFC, priv->ramfc->addr >> 8 | 0x00010000); + + nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH1, priv->base.max); + + nv_wr32(priv, NV03_PFIFO_INTR_0, 0xffffffff); + nv_wr32(priv, NV03_PFIFO_INTR_EN_0, 0xffffffff); + + nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 1); + nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1); + nv_wr32(priv, NV03_PFIFO_CACHES, 1); + return 0; +} + +struct nouveau_oclass * +nv17_fifo_oclass = &(struct nouveau_oclass) { + .handle = NV_ENGINE(FIFO, 0x17), + .ofuncs = &(struct nouveau_ofuncs) { + .ctor = nv17_fifo_ctor, + .dtor = nv04_fifo_dtor, + .init = nv17_fifo_init, + .fini = _nouveau_fifo_fini, + }, +}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c new file mode 100644 index 000000000000..604cb5286632 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c @@ -0,0 +1,361 @@ +/* + * Copyright 2012 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Ben Skeggs + */ + +#include <core/client.h> +#include <nvif/unpack.h> +#include <nvif/class.h> +#include <core/engctx.h> +#include <core/ramht.h> + +#include <subdev/instmem.h> +#include <subdev/instmem/nv04.h> +#include <subdev/fb.h> + +#include <engine/fifo.h> + +#include "nv04.h" + +static struct ramfc_desc +nv40_ramfc[] = { + { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, + { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, + { 32, 0, 0x08, 0, NV10_PFIFO_CACHE1_REF_CNT }, + { 32, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, + { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, + { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_DMA_STATE }, + { 28, 0, 0x18, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, + { 2, 28, 0x18, 28, 0x002058 }, + { 32, 0, 0x1c, 0, NV04_PFIFO_CACHE1_ENGINE }, + { 32, 0, 0x20, 0, NV04_PFIFO_CACHE1_PULL1 }, + { 32, 0, 0x24, 0, NV10_PFIFO_CACHE1_ACQUIRE_VALUE }, + { 32, 0, 0x28, 0, NV10_PFIFO_CACHE1_ACQUIRE_TIMESTAMP }, + { 32, 0, 0x2c, 0, NV10_PFIFO_CACHE1_ACQUIRE_TIMEOUT }, + { 32, 0, 0x30, 0, NV10_PFIFO_CACHE1_SEMAPHORE }, + { 32, 0, 0x34, 0, NV10_PFIFO_CACHE1_DMA_SUBROUTINE }, + { 32, 0, 0x38, 0, NV40_PFIFO_GRCTX_INSTANCE }, + { 17, 0, 0x3c, 0, NV04_PFIFO_DMA_TIMESLICE }, + { 32, 0, 0x40, 0, 0x0032e4 }, + { 32, 0, 0x44, 0, 0x0032e8 }, + { 32, 0, 0x4c, 0, 0x002088 }, + { 32, 0, 0x50, 0, 0x003300 }, + { 32, 0, 0x54, 0, 0x00330c }, + {} +}; + +/******************************************************************************* + * FIFO channel objects + ******************************************************************************/ + +static int +nv40_fifo_object_attach(struct nouveau_object *parent, + struct nouveau_object *object, u32 handle) +{ + struct nv04_fifo_priv *priv = (void *)parent->engine; + struct nv04_fifo_chan *chan = (void *)parent; + u32 context, chid = chan->base.chid; + int ret; + + if (nv_iclass(object, NV_GPUOBJ_CLASS)) + context = nv_gpuobj(object)->addr >> 4; + else + context = 0x00000004; /* just non-zero */ + + switch (nv_engidx(object->engine)) { + case NVDEV_ENGINE_DMAOBJ: + case NVDEV_ENGINE_SW: + context |= 0x00000000; + break; + case NVDEV_ENGINE_GR: + context |= 0x00100000; + break; + case NVDEV_ENGINE_MPEG: + context |= 0x00200000; + break; + default: + return -EINVAL; + } + + context |= chid << 23; + + mutex_lock(&nv_subdev(priv)->mutex); + ret = nouveau_ramht_insert(priv->ramht, chid, handle, context); + mutex_unlock(&nv_subdev(priv)->mutex); + return ret; +} + +static int +nv40_fifo_context_attach(struct nouveau_object *parent, + struct nouveau_object *engctx) +{ + struct nv04_fifo_priv *priv = (void *)parent->engine; + struct nv04_fifo_chan *chan = (void *)parent; + unsigned long flags; + u32 reg, ctx; + + switch (nv_engidx(engctx->engine)) { + case NVDEV_ENGINE_SW: + return 0; + case NVDEV_ENGINE_GR: + reg = 0x32e0; + ctx = 0x38; + break; + case NVDEV_ENGINE_MPEG: + reg = 0x330c; + ctx = 0x54; + break; + default: + return -EINVAL; + } + + spin_lock_irqsave(&priv->base.lock, flags); + nv_engctx(engctx)->addr = nv_gpuobj(engctx)->addr >> 4; + nv_mask(priv, 0x002500, 0x00000001, 0x00000000); + + if ((nv_rd32(priv, 0x003204) & priv->base.max) == chan->base.chid) + nv_wr32(priv, reg, nv_engctx(engctx)->addr); + nv_wo32(priv->ramfc, chan->ramfc + ctx, nv_engctx(engctx)->addr); + + nv_mask(priv, 0x002500, 0x00000001, 0x00000001); + spin_unlock_irqrestore(&priv->base.lock, flags); + return 0; +} + +static int +nv40_fifo_context_detach(struct nouveau_object *parent, bool suspend, + struct nouveau_object *engctx) +{ + struct nv04_fifo_priv *priv = (void *)parent->engine; + struct nv04_fifo_chan *chan = (void *)parent; + unsigned long flags; + u32 reg, ctx; + + switch (nv_engidx(engctx->engine)) { + case NVDEV_ENGINE_SW: + return 0; + case NVDEV_ENGINE_GR: + reg = 0x32e0; + ctx = 0x38; + break; + case NVDEV_ENGINE_MPEG: + reg = 0x330c; + ctx = 0x54; + break; + default: + return -EINVAL; + } + + spin_lock_irqsave(&priv->base.lock, flags); + nv_mask(priv, 0x002500, 0x00000001, 0x00000000); + + if ((nv_rd32(priv, 0x003204) & priv->base.max) == chan->base.chid) + nv_wr32(priv, reg, 0x00000000); + nv_wo32(priv->ramfc, chan->ramfc + ctx, 0x00000000); + + nv_mask(priv, 0x002500, 0x00000001, 0x00000001); + spin_unlock_irqrestore(&priv->base.lock, flags); + return 0; +} + +static int +nv40_fifo_chan_ctor(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + union { + struct nv03_channel_dma_v0 v0; + } *args = data; + struct nv04_fifo_priv *priv = (void *)engine; + struct nv04_fifo_chan *chan; + int ret; + + nv_ioctl(parent, "create channel dma size %d\n", size); + if (nvif_unpack(args->v0, 0, 0, false)) { + nv_ioctl(parent, "create channel dma vers %d pushbuf %08x " + "offset %016llx\n", args->v0.version, + args->v0.pushbuf, args->v0.offset); + } else + return ret; + + ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0xc00000, + 0x1000, args->v0.pushbuf, + (1ULL << NVDEV_ENGINE_DMAOBJ) | + (1ULL << NVDEV_ENGINE_SW) | + (1ULL << NVDEV_ENGINE_GR) | + (1ULL << NVDEV_ENGINE_MPEG), &chan); + *pobject = nv_object(chan); + if (ret) + return ret; + + args->v0.chid = chan->base.chid; + + nv_parent(chan)->context_attach = nv40_fifo_context_attach; + nv_parent(chan)->context_detach = nv40_fifo_context_detach; + nv_parent(chan)->object_attach = nv40_fifo_object_attach; + nv_parent(chan)->object_detach = nv04_fifo_object_detach; + chan->ramfc = chan->base.chid * 128; + + nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->v0.offset); + nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->v0.offset); + nv_wo32(priv->ramfc, chan->ramfc + 0x0c, chan->base.pushgpu->addr >> 4); + nv_wo32(priv->ramfc, chan->ramfc + 0x18, 0x30000000 | + NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES | + NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES | +#ifdef __BIG_ENDIAN + NV_PFIFO_CACHE1_BIG_ENDIAN | +#endif + NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8); + nv_wo32(priv->ramfc, chan->ramfc + 0x3c, 0x0001ffff); + return 0; +} + +static struct nouveau_ofuncs +nv40_fifo_ofuncs = { + .ctor = nv40_fifo_chan_ctor, + .dtor = nv04_fifo_chan_dtor, + .init = nv04_fifo_chan_init, + .fini = nv04_fifo_chan_fini, + .map = _nouveau_fifo_channel_map, + .rd32 = _nouveau_fifo_channel_rd32, + .wr32 = _nouveau_fifo_channel_wr32, + .ntfy = _nouveau_fifo_channel_ntfy +}; + +static struct nouveau_oclass +nv40_fifo_sclass[] = { + { NV40_CHANNEL_DMA, &nv40_fifo_ofuncs }, + {} +}; + +/******************************************************************************* + * FIFO context - basically just the instmem reserved for the channel + ******************************************************************************/ + +static struct nouveau_oclass +nv40_fifo_cclass = { + .handle = NV_ENGCTX(FIFO, 0x40), + .ofuncs = &(struct nouveau_ofuncs) { + .ctor = nv04_fifo_context_ctor, + .dtor = _nouveau_fifo_context_dtor, + .init = _nouveau_fifo_context_init, + .fini = _nouveau_fifo_context_fini, + .rd32 = _nouveau_fifo_context_rd32, + .wr32 = _nouveau_fifo_context_wr32, + }, +}; + +/******************************************************************************* + * PFIFO engine + ******************************************************************************/ + +static int +nv40_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + struct nv04_instmem_priv *imem = nv04_instmem(parent); + struct nv04_fifo_priv *priv; + int ret; + + ret = nouveau_fifo_create(parent, engine, oclass, 0, 31, &priv); + *pobject = nv_object(priv); + if (ret) + return ret; + + nouveau_ramht_ref(imem->ramht, &priv->ramht); + nouveau_gpuobj_ref(imem->ramro, &priv->ramro); + nouveau_gpuobj_ref(imem->ramfc, &priv->ramfc); + + nv_subdev(priv)->unit = 0x00000100; + nv_subdev(priv)->intr = nv04_fifo_intr; + nv_engine(priv)->cclass = &nv40_fifo_cclass; + nv_engine(priv)->sclass = nv40_fifo_sclass; + priv->base.pause = nv04_fifo_pause; + priv->base.start = nv04_fifo_start; + priv->ramfc_desc = nv40_ramfc; + return 0; +} + +static int +nv40_fifo_init(struct nouveau_object *object) +{ + struct nv04_fifo_priv *priv = (void *)object; + struct nouveau_fb *pfb = nouveau_fb(object); + int ret; + + ret = nouveau_fifo_init(&priv->base); + if (ret) + return ret; + + nv_wr32(priv, 0x002040, 0x000000ff); + nv_wr32(priv, 0x002044, 0x2101ffff); + nv_wr32(priv, 0x002058, 0x00000001); + + nv_wr32(priv, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ | + ((priv->ramht->bits - 9) << 16) | + (priv->ramht->gpuobj.addr >> 8)); + nv_wr32(priv, NV03_PFIFO_RAMRO, priv->ramro->addr >> 8); + + switch (nv_device(priv)->chipset) { + case 0x47: + case 0x49: + case 0x4b: + nv_wr32(priv, 0x002230, 0x00000001); + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x45: + case 0x48: + nv_wr32(priv, 0x002220, 0x00030002); + break; + default: + nv_wr32(priv, 0x002230, 0x00000000); + nv_wr32(priv, 0x002220, ((pfb->ram->size - 512 * 1024 + + priv->ramfc->addr) >> 16) | + 0x00030000); + break; + } + + nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH1, priv->base.max); + + nv_wr32(priv, NV03_PFIFO_INTR_0, 0xffffffff); + nv_wr32(priv, NV03_PFIFO_INTR_EN_0, 0xffffffff); + + nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 1); + nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1); + nv_wr32(priv, NV03_PFIFO_CACHES, 1); + return 0; +} + +struct nouveau_oclass * +nv40_fifo_oclass = &(struct nouveau_oclass) { + .handle = NV_ENGINE(FIFO, 0x40), + .ofuncs = &(struct nouveau_ofuncs) { + .ctor = nv40_fifo_ctor, + .dtor = nv04_fifo_dtor, + .init = nv40_fifo_init, + .fini = _nouveau_fifo_fini, + }, +}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c new file mode 100644 index 000000000000..403fafc05072 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -0,0 +1,541 @@ +/* + * Copyright 2012 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Ben Skeggs + */ + +#include <core/client.h> +#include <core/engctx.h> +#include <core/ramht.h> +#include <nvif/unpack.h> +#include <nvif/class.h> + +#include <subdev/timer.h> +#include <subdev/bar.h> + +#include <engine/dmaobj.h> +#include <engine/fifo.h> + +#include "nv04.h" +#include "nv50.h" + +/******************************************************************************* + * FIFO channel objects + ******************************************************************************/ + +static void +nv50_fifo_playlist_update_locked(struct nv50_fifo_priv *priv) +{ + struct nouveau_bar *bar = nouveau_bar(priv); + struct nouveau_gpuobj *cur; + int i, p; + + cur = priv->playlist[priv->cur_playlist]; + priv->cur_playlist = !priv->cur_playlist; + + for (i = priv->base.min, p = 0; i < priv->base.max; i++) { + if (nv_rd32(priv, 0x002600 + (i * 4)) & 0x80000000) + nv_wo32(cur, p++ * 4, i); + } + + bar->flush(bar); + + nv_wr32(priv, 0x0032f4, cur->addr >> 12); + nv_wr32(priv, 0x0032ec, p); + nv_wr32(priv, 0x002500, 0x00000101); +} + +void +nv50_fifo_playlist_update(struct nv50_fifo_priv *priv) +{ + mutex_lock(&nv_subdev(priv)->mutex); + nv50_fifo_playlist_update_locked(priv); + mutex_unlock(&nv_subdev(priv)->mutex); +} + +static int +nv50_fifo_context_attach(struct nouveau_object *parent, + struct nouveau_object *object) +{ + struct nouveau_bar *bar = nouveau_bar(parent); + struct nv50_fifo_base *base = (void *)parent->parent; + struct nouveau_gpuobj *ectx = (void *)object; + u64 limit = ectx->addr + ectx->size - 1; + u64 start = ectx->addr; + u32 addr; + + switch (nv_engidx(object->engine)) { + case NVDEV_ENGINE_SW : return 0; + case NVDEV_ENGINE_GR : addr = 0x0000; break; + case NVDEV_ENGINE_MPEG : addr = 0x0060; break; + default: + return -EINVAL; + } + + nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12; + nv_wo32(base->eng, addr + 0x00, 0x00190000); + nv_wo32(base->eng, addr + 0x04, lower_32_bits(limit)); + nv_wo32(base->eng, addr + 0x08, lower_32_bits(start)); + nv_wo32(base->eng, addr + 0x0c, upper_32_bits(limit) << 24 | + upper_32_bits(start)); + nv_wo32(base->eng, addr + 0x10, 0x00000000); + nv_wo32(base->eng, addr + 0x14, 0x00000000); + bar->flush(bar); + return 0; +} + +static int +nv50_fifo_context_detach(struct nouveau_object *parent, bool suspend, + struct nouveau_object *object) +{ + struct nouveau_bar *bar = nouveau_bar(parent); + struct nv50_fifo_priv *priv = (void *)parent->engine; + struct nv50_fifo_base *base = (void *)parent->parent; + struct nv50_fifo_chan *chan = (void *)parent; + u32 addr, me; + int ret = 0; + + switch (nv_engidx(object->engine)) { + case NVDEV_ENGINE_SW : return 0; + case NVDEV_ENGINE_GR : addr = 0x0000; break; + case NVDEV_ENGINE_MPEG : addr = 0x0060; break; + default: + return -EINVAL; + } + + /* HW bug workaround: + * + * PFIFO will hang forever if the connected engines don't report + * that they've processed the context switch request. + * + * In order for the kickoff to work, we need to ensure all the + * connected engines are in a state where they can answer. + * + * Newer chipsets don't seem to suffer from this issue, and well, + * there's also a "ignore these engines" bitmask reg we can use + * if we hit the issue there.. + */ + me = nv_mask(priv, 0x00b860, 0x00000001, 0x00000001); + + /* do the kickoff... */ + nv_wr32(priv, 0x0032fc, nv_gpuobj(base)->addr >> 12); + if (!nv_wait_ne(priv, 0x0032fc, 0xffffffff, 0xffffffff)) { + nv_error(priv, "channel %d [%s] unload timeout\n", + chan->base.chid, nouveau_client_name(chan)); + if (suspend) + ret = -EBUSY; + } + nv_wr32(priv, 0x00b860, me); + + if (ret == 0) { + nv_wo32(base->eng, addr + 0x00, 0x00000000); + nv_wo32(base->eng, addr + 0x04, 0x00000000); + nv_wo32(base->eng, addr + 0x08, 0x00000000); + nv_wo32(base->eng, addr + 0x0c, 0x00000000); + nv_wo32(base->eng, addr + 0x10, 0x00000000); + nv_wo32(base->eng, addr + 0x14, 0x00000000); + bar->flush(bar); + } + + return ret; +} + +static int +nv50_fifo_object_attach(struct nouveau_object *parent, + struct nouveau_object *object, u32 handle) +{ + struct nv50_fifo_chan *chan = (void *)parent; + u32 context; + + if (nv_iclass(object, NV_GPUOBJ_CLASS)) + context = nv_gpuobj(object)->node->offset >> 4; + else + context = 0x00000004; /* just non-zero */ + + switch (nv_engidx(object->engine)) { + case NVDEV_ENGINE_DMAOBJ: + case NVDEV_ENGINE_SW : context |= 0x00000000; break; + case NVDEV_ENGINE_GR : context |= 0x00100000; break; + case NVDEV_ENGINE_MPEG : context |= 0x00200000; break; + default: + return -EINVAL; + } + + return nouveau_ramht_insert(chan->ramht, 0, handle, context); +} + +void +nv50_fifo_object_detach(struct nouveau_object *parent, int cookie) +{ + struct nv50_fifo_chan *chan = (void *)parent; + nouveau_ramht_remove(chan->ramht, cookie); +} + +static int +nv50_fifo_chan_ctor_dma(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + union { + struct nv03_channel_dma_v0 v0; + } *args = data; + struct nouveau_bar *bar = nouveau_bar(parent); + struct nv50_fifo_base *base = (void *)parent; + struct nv50_fifo_chan *chan; + int ret; + + nv_ioctl(parent, "create channel dma size %d\n", size); + if (nvif_unpack(args->v0, 0, 0, false)) { + nv_ioctl(parent, "create channel dma vers %d pushbuf %08x " + "offset %016llx\n", args->v0.version, + args->v0.pushbuf, args->v0.offset); + } else + return ret; + + ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0xc00000, + 0x2000, args->v0.pushbuf, + (1ULL << NVDEV_ENGINE_DMAOBJ) | + (1ULL << NVDEV_ENGINE_SW) | + (1ULL << NVDEV_ENGINE_GR) | + (1ULL << NVDEV_ENGINE_MPEG), &chan); + *pobject = nv_object(chan); + if (ret) + return ret; + + args->v0.chid = chan->base.chid; + + nv_parent(chan)->context_attach = nv50_fifo_context_attach; + nv_parent(chan)->context_detach = nv50_fifo_context_detach; + nv_parent(chan)->object_attach = nv50_fifo_object_attach; + nv_parent(chan)->object_detach = nv50_fifo_object_detach; + + ret = nouveau_ramht_new(nv_object(chan), nv_object(chan), 0x8000, 16, + &chan->ramht); + if (ret) + return ret; + + nv_wo32(base->ramfc, 0x08, lower_32_bits(args->v0.offset)); + nv_wo32(base->ramfc, 0x0c, upper_32_bits(args->v0.offset)); + nv_wo32(base->ramfc, 0x10, lower_32_bits(args->v0.offset)); + nv_wo32(base->ramfc, 0x14, upper_32_bits(args->v0.offset)); + nv_wo32(base->ramfc, 0x3c, 0x003f6078); + nv_wo32(base->ramfc, 0x44, 0x01003fff); + nv_wo32(base->ramfc, 0x48, chan->base.pushgpu->node->offset >> 4); + nv_wo32(base->ramfc, 0x4c, 0xffffffff); + nv_wo32(base->ramfc, 0x60, 0x7fffffff); + nv_wo32(base->ramfc, 0x78, 0x00000000); + nv_wo32(base->ramfc, 0x7c, 0x30000001); + nv_wo32(base->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) | + (4 << 24) /* SEARCH_FULL */ | + (chan->ramht->gpuobj.node->offset >> 4)); + bar->flush(bar); + return 0; +} + +static int +nv50_fifo_chan_ctor_ind(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + union { + struct nv50_channel_gpfifo_v0 v0; + } *args = data; + struct nouveau_bar *bar = nouveau_bar(parent); + struct nv50_fifo_base *base = (void *)parent; + struct nv50_fifo_chan *chan; + u64 ioffset, ilength; + int ret; + + nv_ioctl(parent, "create channel gpfifo size %d\n", size); + if (nvif_unpack(args->v0, 0, 0, false)) { + nv_ioctl(parent, "create channel gpfifo vers %d pushbuf %08x " + "ioffset %016llx ilength %08x\n", + args->v0.version, args->v0.pushbuf, args->v0.ioffset, + args->v0.ilength); + } else + return ret; + + ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0xc00000, + 0x2000, args->v0.pushbuf, + (1ULL << NVDEV_ENGINE_DMAOBJ) | + (1ULL << NVDEV_ENGINE_SW) | + (1ULL << NVDEV_ENGINE_GR) | + (1ULL << NVDEV_ENGINE_MPEG), &chan); + *pobject = nv_object(chan); + if (ret) + return ret; + + args->v0.chid = chan->base.chid; + + nv_parent(chan)->context_attach = nv50_fifo_context_attach; + nv_parent(chan)->context_detach = nv50_fifo_context_detach; + nv_parent(chan)->object_attach = nv50_fifo_object_attach; + nv_parent(chan)->object_detach = nv50_fifo_object_detach; + + ret = nouveau_ramht_new(nv_object(chan), nv_object(chan), 0x8000, 16, + &chan->ramht); + if (ret) + return ret; + + ioffset = args->v0.ioffset; + ilength = order_base_2(args->v0.ilength / 8); + + nv_wo32(base->ramfc, 0x3c, 0x403f6078); + nv_wo32(base->ramfc, 0x44, 0x01003fff); + nv_wo32(base->ramfc, 0x48, chan->base.pushgpu->node->offset >> 4); + nv_wo32(base->ramfc, 0x50, lower_32_bits(ioffset)); + nv_wo32(base->ramfc, 0x54, upper_32_bits(ioffset) | (ilength << 16)); + nv_wo32(base->ramfc, 0x60, 0x7fffffff); + nv_wo32(base->ramfc, 0x78, 0x00000000); + nv_wo32(base->ramfc, 0x7c, 0x30000001); + nv_wo32(base->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) | + (4 << 24) /* SEARCH_FULL */ | + (chan->ramht->gpuobj.node->offset >> 4)); + bar->flush(bar); + return 0; +} + +void +nv50_fifo_chan_dtor(struct nouveau_object *object) +{ + struct nv50_fifo_chan *chan = (void *)object; + nouveau_ramht_ref(NULL, &chan->ramht); + nouveau_fifo_channel_destroy(&chan->base); +} + +static int +nv50_fifo_chan_init(struct nouveau_object *object) +{ + struct nv50_fifo_priv *priv = (void *)object->engine; + struct nv50_fifo_base *base = (void *)object->parent; + struct nv50_fifo_chan *chan = (void *)object; + struct nouveau_gpuobj *ramfc = base->ramfc; + u32 chid = chan->base.chid; + int ret; + + ret = nouveau_fifo_channel_init(&chan->base); + if (ret) + return ret; + + nv_wr32(priv, 0x002600 + (chid * 4), 0x80000000 | ramfc->addr >> 12); + nv50_fifo_playlist_update(priv); + return 0; +} + +int +nv50_fifo_chan_fini(struct nouveau_object *object, bool suspend) +{ + struct nv50_fifo_priv *priv = (void *)object->engine; + struct nv50_fifo_chan *chan = (void *)object; + u32 chid = chan->base.chid; + + /* remove channel from playlist, fifo will unload context */ + nv_mask(priv, 0x002600 + (chid * 4), 0x80000000, 0x00000000); + nv50_fifo_playlist_update(priv); + nv_wr32(priv, 0x002600 + (chid * 4), 0x00000000); + + return nouveau_fifo_channel_fini(&chan->base, suspend); +} + +static struct nouveau_ofuncs +nv50_fifo_ofuncs_dma = { + .ctor = nv50_fifo_chan_ctor_dma, + .dtor = nv50_fifo_chan_dtor, + .init = nv50_fifo_chan_init, + .fini = nv50_fifo_chan_fini, + .map = _nouveau_fifo_channel_map, + .rd32 = _nouveau_fifo_channel_rd32, + .wr32 = _nouveau_fifo_channel_wr32, + .ntfy = _nouveau_fifo_channel_ntfy +}; + +static struct nouveau_ofuncs +nv50_fifo_ofuncs_ind = { + .ctor = nv50_fifo_chan_ctor_ind, + .dtor = nv50_fifo_chan_dtor, + .init = nv50_fifo_chan_init, + .fini = nv50_fifo_chan_fini, + .map = _nouveau_fifo_channel_map, + .rd32 = _nouveau_fifo_channel_rd32, + .wr32 = _nouveau_fifo_channel_wr32, + .ntfy = _nouveau_fifo_channel_ntfy +}; + +static struct nouveau_oclass +nv50_fifo_sclass[] = { + { NV50_CHANNEL_DMA, &nv50_fifo_ofuncs_dma }, + { NV50_CHANNEL_GPFIFO, &nv50_fifo_ofuncs_ind }, + {} +}; + +/******************************************************************************* + * FIFO context - basically just the instmem reserved for the channel + ******************************************************************************/ + +static int +nv50_fifo_context_ctor(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + struct nv50_fifo_base *base; + int ret; + + ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x10000, + 0x1000, NVOBJ_FLAG_HEAP, &base); + *pobject = nv_object(base); + if (ret) + return ret; + + ret = nouveau_gpuobj_new(nv_object(base), nv_object(base), 0x0200, + 0x1000, NVOBJ_FLAG_ZERO_ALLOC, &base->ramfc); + if (ret) + return ret; + + ret = nouveau_gpuobj_new(nv_object(base), nv_object(base), 0x1200, 0, + NVOBJ_FLAG_ZERO_ALLOC, &base->eng); + if (ret) + return ret; + + ret = nouveau_gpuobj_new(nv_object(base), nv_object(base), 0x4000, 0, 0, + &base->pgd); + if (ret) + return ret; + + ret = nouveau_vm_ref(nouveau_client(parent)->vm, &base->vm, base->pgd); + if (ret) + return ret; + + return 0; +} + +void +nv50_fifo_context_dtor(struct nouveau_object *object) +{ + struct nv50_fifo_base *base = (void *)object; + nouveau_vm_ref(NULL, &base->vm, base->pgd); + nouveau_gpuobj_ref(NULL, &base->pgd); + nouveau_gpuobj_ref(NULL, &base->eng); + nouveau_gpuobj_ref(NULL, &base->ramfc); + nouveau_gpuobj_ref(NULL, &base->cache); + nouveau_fifo_context_destroy(&base->base); +} + +static struct nouveau_oclass +nv50_fifo_cclass = { + .handle = NV_ENGCTX(FIFO, 0x50), + .ofuncs = &(struct nouveau_ofuncs) { + .ctor = nv50_fifo_context_ctor, + .dtor = nv50_fifo_context_dtor, + .init = _nouveau_fifo_context_init, + .fini = _nouveau_fifo_context_fini, + .rd32 = _nouveau_fifo_context_rd32, + .wr32 = _nouveau_fifo_context_wr32, + }, +}; + +/******************************************************************************* + * PFIFO engine + ******************************************************************************/ + +static int +nv50_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + struct nv50_fifo_priv *priv; + int ret; + + ret = nouveau_fifo_create(parent, engine, oclass, 1, 127, &priv); + *pobject = nv_object(priv); + if (ret) + return ret; + + ret = nouveau_gpuobj_new(nv_object(priv), NULL, 128 * 4, 0x1000, 0, + &priv->playlist[0]); + if (ret) + return ret; + + ret = nouveau_gpuobj_new(nv_object(priv), NULL, 128 * 4, 0x1000, 0, + &priv->playlist[1]); + if (ret) + return ret; + + nv_subdev(priv)->unit = 0x00000100; + nv_subdev(priv)->intr = nv04_fifo_intr; + nv_engine(priv)->cclass = &nv50_fifo_cclass; + nv_engine(priv)->sclass = nv50_fifo_sclass; + priv->base.pause = nv04_fifo_pause; + priv->base.start = nv04_fifo_start; + return 0; +} + +void +nv50_fifo_dtor(struct nouveau_object *object) +{ + struct nv50_fifo_priv *priv = (void *)object; + + nouveau_gpuobj_ref(NULL, &priv->playlist[1]); + nouveau_gpuobj_ref(NULL, &priv->playlist[0]); + + nouveau_fifo_destroy(&priv->base); +} + +int +nv50_fifo_init(struct nouveau_object *object) +{ + struct nv50_fifo_priv *priv = (void *)object; + int ret, i; + + ret = nouveau_fifo_init(&priv->base); + if (ret) + return ret; + + nv_mask(priv, 0x000200, 0x00000100, 0x00000000); + nv_mask(priv, 0x000200, 0x00000100, 0x00000100); + nv_wr32(priv, 0x00250c, 0x6f3cfc34); + nv_wr32(priv, 0x002044, 0x01003fff); + + nv_wr32(priv, 0x002100, 0xffffffff); + nv_wr32(priv, 0x002140, 0xbfffffff); + + for (i = 0; i < 128; i++) + nv_wr32(priv, 0x002600 + (i * 4), 0x00000000); + nv50_fifo_playlist_update_locked(priv); + + nv_wr32(priv, 0x003200, 0x00000001); + nv_wr32(priv, 0x003250, 0x00000001); + nv_wr32(priv, 0x002500, 0x00000001); + return 0; +} + +struct nouveau_oclass * +nv50_fifo_oclass = &(struct nouveau_oclass) { + .handle = NV_ENGINE(FIFO, 0x50), + .ofuncs = &(struct nouveau_ofuncs) { + .ctor = nv50_fifo_ctor, + .dtor = nv50_fifo_dtor, + .init = nv50_fifo_init, + .fini = _nouveau_fifo_fini, + }, +}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h new file mode 100644 index 000000000000..3a9ceb315c20 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h @@ -0,0 +1,36 @@ +#ifndef __NV50_FIFO_H__ +#define __NV50_FIFO_H__ + +struct nv50_fifo_priv { + struct nouveau_fifo base; + struct nouveau_gpuobj *playlist[2]; + int cur_playlist; +}; + +struct nv50_fifo_base { + struct nouveau_fifo_base base; + struct nouveau_gpuobj *ramfc; + struct nouveau_gpuobj *cache; + struct nouveau_gpuobj *eng; + struct nouveau_gpuobj *pgd; + struct nouveau_vm *vm; +}; + +struct nv50_fifo_chan { + struct nouveau_fifo_chan base; + u32 subc[8]; + struct nouveau_ramht *ramht; +}; + +void nv50_fifo_playlist_update(struct nv50_fifo_priv *); + +void nv50_fifo_object_detach(struct nouveau_object *, int); +void nv50_fifo_chan_dtor(struct nouveau_object *); +int nv50_fifo_chan_fini(struct nouveau_object *, bool); + +void nv50_fifo_context_dtor(struct nouveau_object *); + +void nv50_fifo_dtor(struct nouveau_object *); +int nv50_fifo_init(struct nouveau_object *); + +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv84.c new file mode 100644 index 000000000000..b18386b8a842 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv84.c @@ -0,0 +1,481 @@ +/* + * Copyright 2012 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Ben Skeggs + */ + +#include <core/os.h> +#include <core/client.h> +#include <core/engctx.h> +#include <core/ramht.h> +#include <core/event.h> +#include <nvif/unpack.h> +#include <nvif/class.h> + +#include <subdev/timer.h> +#include <subdev/bar.h> + +#include <engine/dmaobj.h> +#include <engine/fifo.h> + +#include "nv04.h" +#include "nv50.h" + +/******************************************************************************* + * FIFO channel objects + ******************************************************************************/ + +static int +nv84_fifo_context_attach(struct nouveau_object *parent, + struct nouveau_object *object) +{ + struct nouveau_bar *bar = nouveau_bar(parent); + struct nv50_fifo_base *base = (void *)parent->parent; + struct nouveau_gpuobj *ectx = (void *)object; + u64 limit = ectx->addr + ectx->size - 1; + u64 start = ectx->addr; + u32 addr; + + switch (nv_engidx(object->engine)) { + case NVDEV_ENGINE_SW : return 0; + case NVDEV_ENGINE_GR : addr = 0x0020; break; + case NVDEV_ENGINE_VP : addr = 0x0040; break; + case NVDEV_ENGINE_PPP : + case NVDEV_ENGINE_MPEG : addr = 0x0060; break; + case NVDEV_ENGINE_BSP : addr = 0x0080; break; + case NVDEV_ENGINE_CRYPT: addr = 0x00a0; break; + case NVDEV_ENGINE_COPY0: addr = 0x00c0; break; + default: + return -EINVAL; + } + + nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12; + nv_wo32(base->eng, addr + 0x00, 0x00190000); + nv_wo32(base->eng, addr + 0x04, lower_32_bits(limit)); + nv_wo32(base->eng, addr + 0x08, lower_32_bits(start)); + nv_wo32(base->eng, addr + 0x0c, upper_32_bits(limit) << 24 | + upper_32_bits(start)); + nv_wo32(base->eng, addr + 0x10, 0x00000000); + nv_wo32(base->eng, addr + 0x14, 0x00000000); + bar->flush(bar); + return 0; +} + +static int +nv84_fifo_context_detach(struct nouveau_object *parent, bool suspend, + struct nouveau_object *object) +{ + struct nouveau_bar *bar = nouveau_bar(parent); + struct nv50_fifo_priv *priv = (void *)parent->engine; + struct nv50_fifo_base *base = (void *)parent->parent; + struct nv50_fifo_chan *chan = (void *)parent; + u32 addr, save, engn; + bool done; + + switch (nv_engidx(object->engine)) { + case NVDEV_ENGINE_SW : return 0; + case NVDEV_ENGINE_GR : engn = 0; addr = 0x0020; break; + case NVDEV_ENGINE_VP : engn = 3; addr = 0x0040; break; + case NVDEV_ENGINE_PPP : + case NVDEV_ENGINE_MPEG : engn = 1; addr = 0x0060; break; + case NVDEV_ENGINE_BSP : engn = 5; addr = 0x0080; break; + case NVDEV_ENGINE_CRYPT: engn = 4; addr = 0x00a0; break; + case NVDEV_ENGINE_COPY0: engn = 2; addr = 0x00c0; break; + default: + return -EINVAL; + } + + save = nv_mask(priv, 0x002520, 0x0000003f, 1 << engn); + nv_wr32(priv, 0x0032fc, nv_gpuobj(base)->addr >> 12); + done = nv_wait_ne(priv, 0x0032fc, 0xffffffff, 0xffffffff); + nv_wr32(priv, 0x002520, save); + if (!done) { + nv_error(priv, "channel %d [%s] unload timeout\n", + chan->base.chid, nouveau_client_name(chan)); + if (suspend) + return -EBUSY; + } + + nv_wo32(base->eng, addr + 0x00, 0x00000000); + nv_wo32(base->eng, addr + 0x04, 0x00000000); + nv_wo32(base->eng, addr + 0x08, 0x00000000); + nv_wo32(base->eng, addr + 0x0c, 0x00000000); + nv_wo32(base->eng, addr + 0x10, 0x00000000); + nv_wo32(base->eng, addr + 0x14, 0x00000000); + bar->flush(bar); + return 0; +} + +static int +nv84_fifo_object_attach(struct nouveau_object *parent, + struct nouveau_object *object, u32 handle) +{ + struct nv50_fifo_chan *chan = (void *)parent; + u32 context; + + if (nv_iclass(object, NV_GPUOBJ_CLASS)) + context = nv_gpuobj(object)->node->offset >> 4; + else + context = 0x00000004; /* just non-zero */ + + switch (nv_engidx(object->engine)) { + case NVDEV_ENGINE_DMAOBJ: + case NVDEV_ENGINE_SW : context |= 0x00000000; break; + case NVDEV_ENGINE_GR : context |= 0x00100000; break; + case NVDEV_ENGINE_MPEG : + case NVDEV_ENGINE_PPP : context |= 0x00200000; break; + case NVDEV_ENGINE_ME : + case NVDEV_ENGINE_COPY0 : context |= 0x00300000; break; + case NVDEV_ENGINE_VP : context |= 0x00400000; break; + case NVDEV_ENGINE_CRYPT : + case NVDEV_ENGINE_VIC : context |= 0x00500000; break; + case NVDEV_ENGINE_BSP : context |= 0x00600000; break; + default: + return -EINVAL; + } + + return nouveau_ramht_insert(chan->ramht, 0, handle, context); +} + +static int +nv84_fifo_chan_ctor_dma(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + union { + struct nv03_channel_dma_v0 v0; + } *args = data; + struct nouveau_bar *bar = nouveau_bar(parent); + struct nv50_fifo_base *base = (void *)parent; + struct nv50_fifo_chan *chan; + int ret; + + nv_ioctl(parent, "create channel dma size %d\n", size); + if (nvif_unpack(args->v0, 0, 0, false)) { + nv_ioctl(parent, "create channel dma vers %d pushbuf %08x " + "offset %016llx\n", args->v0.version, + args->v0.pushbuf, args->v0.offset); + } else + return ret; + + ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0xc00000, + 0x2000, args->v0.pushbuf, + (1ULL << NVDEV_ENGINE_DMAOBJ) | + (1ULL << NVDEV_ENGINE_SW) | + (1ULL << NVDEV_ENGINE_GR) | + (1ULL << NVDEV_ENGINE_MPEG) | + (1ULL << NVDEV_ENGINE_ME) | + (1ULL << NVDEV_ENGINE_VP) | + (1ULL << NVDEV_ENGINE_CRYPT) | + (1ULL << NVDEV_ENGINE_BSP) | + (1ULL << NVDEV_ENGINE_PPP) | + (1ULL << NVDEV_ENGINE_COPY0) | + (1ULL << NVDEV_ENGINE_VIC), &chan); + *pobject = nv_object(chan); + if (ret) + return ret; + + args->v0.chid = chan->base.chid; + + ret = nouveau_ramht_new(nv_object(chan), nv_object(chan), 0x8000, 16, + &chan->ramht); + if (ret) + return ret; + + nv_parent(chan)->context_attach = nv84_fifo_context_attach; + nv_parent(chan)->context_detach = nv84_fifo_context_detach; + nv_parent(chan)->object_attach = nv84_fifo_object_attach; + nv_parent(chan)->object_detach = nv50_fifo_object_detach; + + nv_wo32(base->ramfc, 0x08, lower_32_bits(args->v0.offset)); + nv_wo32(base->ramfc, 0x0c, upper_32_bits(args->v0.offset)); + nv_wo32(base->ramfc, 0x10, lower_32_bits(args->v0.offset)); + nv_wo32(base->ramfc, 0x14, upper_32_bits(args->v0.offset)); + nv_wo32(base->ramfc, 0x3c, 0x003f6078); + nv_wo32(base->ramfc, 0x44, 0x01003fff); + nv_wo32(base->ramfc, 0x48, chan->base.pushgpu->node->offset >> 4); + nv_wo32(base->ramfc, 0x4c, 0xffffffff); + nv_wo32(base->ramfc, 0x60, 0x7fffffff); + nv_wo32(base->ramfc, 0x78, 0x00000000); + nv_wo32(base->ramfc, 0x7c, 0x30000001); + nv_wo32(base->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) | + (4 << 24) /* SEARCH_FULL */ | + (chan->ramht->gpuobj.node->offset >> 4)); + nv_wo32(base->ramfc, 0x88, base->cache->addr >> 10); + nv_wo32(base->ramfc, 0x98, nv_gpuobj(base)->addr >> 12); + bar->flush(bar); + return 0; +} + +static int +nv84_fifo_chan_ctor_ind(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + union { + struct nv50_channel_gpfifo_v0 v0; + } *args = data; + struct nouveau_bar *bar = nouveau_bar(parent); + struct nv50_fifo_base *base = (void *)parent; + struct nv50_fifo_chan *chan; + u64 ioffset, ilength; + int ret; + + nv_ioctl(parent, "create channel gpfifo size %d\n", size); + if (nvif_unpack(args->v0, 0, 0, false)) { + nv_ioctl(parent, "create channel gpfifo vers %d pushbuf %08x " + "ioffset %016llx ilength %08x\n", + args->v0.version, args->v0.pushbuf, args->v0.ioffset, + args->v0.ilength); + } else + return ret; + + ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0xc00000, + 0x2000, args->v0.pushbuf, + (1ULL << NVDEV_ENGINE_DMAOBJ) | + (1ULL << NVDEV_ENGINE_SW) | + (1ULL << NVDEV_ENGINE_GR) | + (1ULL << NVDEV_ENGINE_MPEG) | + (1ULL << NVDEV_ENGINE_ME) | + (1ULL << NVDEV_ENGINE_VP) | + (1ULL << NVDEV_ENGINE_CRYPT) | + (1ULL << NVDEV_ENGINE_BSP) | + (1ULL << NVDEV_ENGINE_PPP) | + (1ULL << NVDEV_ENGINE_COPY0) | + (1ULL << NVDEV_ENGINE_VIC), &chan); + *pobject = nv_object(chan); + if (ret) + return ret; + + args->v0.chid = chan->base.chid; + + ret = nouveau_ramht_new(nv_object(chan), nv_object(chan), 0x8000, 16, + &chan->ramht); + if (ret) + return ret; + + nv_parent(chan)->context_attach = nv84_fifo_context_attach; + nv_parent(chan)->context_detach = nv84_fifo_context_detach; + nv_parent(chan)->object_attach = nv84_fifo_object_attach; + nv_parent(chan)->object_detach = nv50_fifo_object_detach; + + ioffset = args->v0.ioffset; + ilength = order_base_2(args->v0.ilength / 8); + + nv_wo32(base->ramfc, 0x3c, 0x403f6078); + nv_wo32(base->ramfc, 0x44, 0x01003fff); + nv_wo32(base->ramfc, 0x48, chan->base.pushgpu->node->offset >> 4); + nv_wo32(base->ramfc, 0x50, lower_32_bits(ioffset)); + nv_wo32(base->ramfc, 0x54, upper_32_bits(ioffset) | (ilength << 16)); + nv_wo32(base->ramfc, 0x60, 0x7fffffff); + nv_wo32(base->ramfc, 0x78, 0x00000000); + nv_wo32(base->ramfc, 0x7c, 0x30000001); + nv_wo32(base->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) | + (4 << 24) /* SEARCH_FULL */ | + (chan->ramht->gpuobj.node->offset >> 4)); + nv_wo32(base->ramfc, 0x88, base->cache->addr >> 10); + nv_wo32(base->ramfc, 0x98, nv_gpuobj(base)->addr >> 12); + bar->flush(bar); + return 0; +} + +static int +nv84_fifo_chan_init(struct nouveau_object *object) +{ + struct nv50_fifo_priv *priv = (void *)object->engine; + struct nv50_fifo_base *base = (void *)object->parent; + struct nv50_fifo_chan *chan = (void *)object; + struct nouveau_gpuobj *ramfc = base->ramfc; + u32 chid = chan->base.chid; + int ret; + + ret = nouveau_fifo_channel_init(&chan->base); + if (ret) + return ret; + + nv_wr32(priv, 0x002600 + (chid * 4), 0x80000000 | ramfc->addr >> 8); + nv50_fifo_playlist_update(priv); + return 0; +} + +static struct nouveau_ofuncs +nv84_fifo_ofuncs_dma = { + .ctor = nv84_fifo_chan_ctor_dma, + .dtor = nv50_fifo_chan_dtor, + .init = nv84_fifo_chan_init, + .fini = nv50_fifo_chan_fini, + .map = _nouveau_fifo_channel_map, + .rd32 = _nouveau_fifo_channel_rd32, + .wr32 = _nouveau_fifo_channel_wr32, + .ntfy = _nouveau_fifo_channel_ntfy +}; + +static struct nouveau_ofuncs +nv84_fifo_ofuncs_ind = { + .ctor = nv84_fifo_chan_ctor_ind, + .dtor = nv50_fifo_chan_dtor, + .init = nv84_fifo_chan_init, + .fini = nv50_fifo_chan_fini, + .map = _nouveau_fifo_channel_map, + .rd32 = _nouveau_fifo_channel_rd32, + .wr32 = _nouveau_fifo_channel_wr32, + .ntfy = _nouveau_fifo_channel_ntfy +}; + +static struct nouveau_oclass +nv84_fifo_sclass[] = { + { G82_CHANNEL_DMA, &nv84_fifo_ofuncs_dma }, + { G82_CHANNEL_GPFIFO, &nv84_fifo_ofuncs_ind }, + {} +}; + +/******************************************************************************* + * FIFO context - basically just the instmem reserved for the channel + ******************************************************************************/ + +static int +nv84_fifo_context_ctor(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + struct nv50_fifo_base *base; + int ret; + + ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x10000, + 0x1000, NVOBJ_FLAG_HEAP, &base); + *pobject = nv_object(base); + if (ret) + return ret; + + ret = nouveau_gpuobj_new(nv_object(base), nv_object(base), 0x0200, 0, + NVOBJ_FLAG_ZERO_ALLOC, &base->eng); + if (ret) + return ret; + + ret = nouveau_gpuobj_new(nv_object(base), nv_object(base), 0x4000, 0, + 0, &base->pgd); + if (ret) + return ret; + + ret = nouveau_vm_ref(nouveau_client(parent)->vm, &base->vm, base->pgd); + if (ret) + return ret; + + ret = nouveau_gpuobj_new(nv_object(base), nv_object(base), 0x1000, + 0x400, NVOBJ_FLAG_ZERO_ALLOC, &base->cache); + if (ret) + return ret; + + ret = nouveau_gpuobj_new(nv_object(base), nv_object(base), 0x0100, + 0x100, NVOBJ_FLAG_ZERO_ALLOC, &base->ramfc); + if (ret) + return ret; + + return 0; +} + +static struct nouveau_oclass +nv84_fifo_cclass = { + .handle = NV_ENGCTX(FIFO, 0x84), + .ofuncs = &(struct nouveau_ofuncs) { + .ctor = nv84_fifo_context_ctor, + .dtor = nv50_fifo_context_dtor, + .init = _nouveau_fifo_context_init, + .fini = _nouveau_fifo_context_fini, + .rd32 = _nouveau_fifo_context_rd32, + .wr32 = _nouveau_fifo_context_wr32, + }, +}; + +/******************************************************************************* + * PFIFO engine + ******************************************************************************/ + +static void +nv84_fifo_uevent_init(struct nvkm_event *event, int type, int index) +{ + struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent); + nv_mask(fifo, 0x002140, 0x40000000, 0x40000000); +} + +static void +nv84_fifo_uevent_fini(struct nvkm_event *event, int type, int index) +{ + struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent); + nv_mask(fifo, 0x002140, 0x40000000, 0x00000000); +} + +static const struct nvkm_event_func +nv84_fifo_uevent_func = { + .ctor = nouveau_fifo_uevent_ctor, + .init = nv84_fifo_uevent_init, + .fini = nv84_fifo_uevent_fini, +}; + +static int +nv84_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + struct nv50_fifo_priv *priv; + int ret; + + ret = nouveau_fifo_create(parent, engine, oclass, 1, 127, &priv); + *pobject = nv_object(priv); + if (ret) + return ret; + + ret = nouveau_gpuobj_new(nv_object(priv), NULL, 128 * 4, 0x1000, 0, + &priv->playlist[0]); + if (ret) + return ret; + + ret = nouveau_gpuobj_new(nv_object(priv), NULL, 128 * 4, 0x1000, 0, + &priv->playlist[1]); + if (ret) + return ret; + + ret = nvkm_event_init(&nv84_fifo_uevent_func, 1, 1, &priv->base.uevent); + if (ret) + return ret; + + nv_subdev(priv)->unit = 0x00000100; + nv_subdev(priv)->intr = nv04_fifo_intr; + nv_engine(priv)->cclass = &nv84_fifo_cclass; + nv_engine(priv)->sclass = nv84_fifo_sclass; + priv->base.pause = nv04_fifo_pause; + priv->base.start = nv04_fifo_start; + return 0; +} + +struct nouveau_oclass * +nv84_fifo_oclass = &(struct nouveau_oclass) { + .handle = NV_ENGINE(FIFO, 0x84), + .ofuncs = &(struct nouveau_ofuncs) { + .ctor = nv84_fifo_ctor, + .dtor = nv50_fifo_dtor, + .init = nv50_fifo_init, + .fini = _nouveau_fifo_fini, + }, +}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nvc0.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nvc0.c new file mode 100644 index 000000000000..e8ef8cb058cd --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nvc0.c @@ -0,0 +1,974 @@ +/* + * Copyright 2012 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Ben Skeggs + */ + +#include <core/client.h> +#include <core/handle.h> +#include <core/namedb.h> +#include <core/gpuobj.h> +#include <core/engctx.h> +#include <core/event.h> +#include <nvif/unpack.h> +#include <nvif/class.h> +#include <core/enum.h> + +#include <subdev/timer.h> +#include <subdev/bar.h> +#include <subdev/fb.h> +#include <subdev/vm.h> + +#include <engine/dmaobj.h> +#include <engine/fifo.h> + +struct nvc0_fifo_priv { + struct nouveau_fifo base; + + struct work_struct fault; + u64 mask; + + struct { + struct nouveau_gpuobj *mem[2]; + int active; + wait_queue_head_t wait; + } runlist; + + struct { + struct nouveau_gpuobj *mem; + struct nouveau_vma bar; + } user; + int spoon_nr; +}; + +struct nvc0_fifo_base { + struct nouveau_fifo_base base; + struct nouveau_gpuobj *pgd; + struct nouveau_vm *vm; +}; + +struct nvc0_fifo_chan { + struct nouveau_fifo_chan base; + enum { + STOPPED, + RUNNING, + KILLED + } state; +}; + +/******************************************************************************* + * FIFO channel objects + ******************************************************************************/ + +static void +nvc0_fifo_runlist_update(struct nvc0_fifo_priv *priv) +{ + struct nouveau_bar *bar = nouveau_bar(priv); + struct nouveau_gpuobj *cur; + int i, p; + + mutex_lock(&nv_subdev(priv)->mutex); + cur = priv->runlist.mem[priv->runlist.active]; + priv->runlist.active = !priv->runlist.active; + + for (i = 0, p = 0; i < 128; i++) { + struct nvc0_fifo_chan *chan = (void *)priv->base.channel[i]; + if (chan && chan->state == RUNNING) { + nv_wo32(cur, p + 0, i); + nv_wo32(cur, p + 4, 0x00000004); + p += 8; + } + } + bar->flush(bar); + + nv_wr32(priv, 0x002270, cur->addr >> 12); + nv_wr32(priv, 0x002274, 0x01f00000 | (p >> 3)); + + if (wait_event_timeout(priv->runlist.wait, + !(nv_rd32(priv, 0x00227c) & 0x00100000), + msecs_to_jiffies(2000)) == 0) + nv_error(priv, "runlist update timeout\n"); + mutex_unlock(&nv_subdev(priv)->mutex); +} + +static int +nvc0_fifo_context_attach(struct nouveau_object *parent, + struct nouveau_object *object) +{ + struct nouveau_bar *bar = nouveau_bar(parent); + struct nvc0_fifo_base *base = (void *)parent->parent; + struct nouveau_engctx *ectx = (void *)object; + u32 addr; + int ret; + + switch (nv_engidx(object->engine)) { + case NVDEV_ENGINE_SW : return 0; + case NVDEV_ENGINE_GR : addr = 0x0210; break; + case NVDEV_ENGINE_COPY0: addr = 0x0230; break; + case NVDEV_ENGINE_COPY1: addr = 0x0240; break; + case NVDEV_ENGINE_BSP : addr = 0x0270; break; + case NVDEV_ENGINE_VP : addr = 0x0250; break; + case NVDEV_ENGINE_PPP : addr = 0x0260; break; + default: + return -EINVAL; + } + + if (!ectx->vma.node) { + ret = nouveau_gpuobj_map_vm(nv_gpuobj(ectx), base->vm, + NV_MEM_ACCESS_RW, &ectx->vma); + if (ret) + return ret; + + nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12; + } + + nv_wo32(base, addr + 0x00, lower_32_bits(ectx->vma.offset) | 4); + nv_wo32(base, addr + 0x04, upper_32_bits(ectx->vma.offset)); + bar->flush(bar); + return 0; +} + +static int +nvc0_fifo_context_detach(struct nouveau_object *parent, bool suspend, + struct nouveau_object *object) +{ + struct nouveau_bar *bar = nouveau_bar(parent); + struct nvc0_fifo_priv *priv = (void *)parent->engine; + struct nvc0_fifo_base *base = (void *)parent->parent; + struct nvc0_fifo_chan *chan = (void *)parent; + u32 addr; + + switch (nv_engidx(object->engine)) { + case NVDEV_ENGINE_SW : return 0; + case NVDEV_ENGINE_GR : addr = 0x0210; break; + case NVDEV_ENGINE_COPY0: addr = 0x0230; break; + case NVDEV_ENGINE_COPY1: addr = 0x0240; break; + case NVDEV_ENGINE_BSP : addr = 0x0270; break; + case NVDEV_ENGINE_VP : addr = 0x0250; break; + case NVDEV_ENGINE_PPP : addr = 0x0260; break; + default: + return -EINVAL; + } + + nv_wr32(priv, 0x002634, chan->base.chid); + if (!nv_wait(priv, 0x002634, 0xffffffff, chan->base.chid)) { + nv_error(priv, "channel %d [%s] kick timeout\n", + chan->base.chid, nouveau_client_name(chan)); + if (suspend) + return -EBUSY; + } + + nv_wo32(base, addr + 0x00, 0x00000000); + nv_wo32(base, addr + 0x04, 0x00000000); + bar->flush(bar); + return 0; +} + +static int +nvc0_fifo_chan_ctor(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + union { + struct nv50_channel_gpfifo_v0 v0; + } *args = data; + struct nouveau_bar *bar = nouveau_bar(parent); + struct nvc0_fifo_priv *priv = (void *)engine; + struct nvc0_fifo_base *base = (void *)parent; + struct nvc0_fifo_chan *chan; + u64 usermem, ioffset, ilength; + int ret, i; + + nv_ioctl(parent, "create channel gpfifo size %d\n", size); + if (nvif_unpack(args->v0, 0, 0, false)) { + nv_ioctl(parent, "create channel gpfifo vers %d pushbuf %08x " + "ioffset %016llx ilength %08x\n", + args->v0.version, args->v0.pushbuf, args->v0.ioffset, + args->v0.ilength); + } else + return ret; + + ret = nouveau_fifo_channel_create(parent, engine, oclass, 1, + priv->user.bar.offset, 0x1000, + args->v0.pushbuf, + (1ULL << NVDEV_ENGINE_SW) | + (1ULL << NVDEV_ENGINE_GR) | + (1ULL << NVDEV_ENGINE_COPY0) | + (1ULL << NVDEV_ENGINE_COPY1) | + (1ULL << NVDEV_ENGINE_BSP) | + (1ULL << NVDEV_ENGINE_VP) | + (1ULL << NVDEV_ENGINE_PPP), &chan); + *pobject = nv_object(chan); + if (ret) + return ret; + + args->v0.chid = chan->base.chid; + + nv_parent(chan)->context_attach = nvc0_fifo_context_attach; + nv_parent(chan)->context_detach = nvc0_fifo_context_detach; + + usermem = chan->base.chid * 0x1000; + ioffset = args->v0.ioffset; + ilength = order_base_2(args->v0.ilength / 8); + + for (i = 0; i < 0x1000; i += 4) + nv_wo32(priv->user.mem, usermem + i, 0x00000000); + + nv_wo32(base, 0x08, lower_32_bits(priv->user.mem->addr + usermem)); + nv_wo32(base, 0x0c, upper_32_bits(priv->user.mem->addr + usermem)); + nv_wo32(base, 0x10, 0x0000face); + nv_wo32(base, 0x30, 0xfffff902); + nv_wo32(base, 0x48, lower_32_bits(ioffset)); + nv_wo32(base, 0x4c, upper_32_bits(ioffset) | (ilength << 16)); + nv_wo32(base, 0x54, 0x00000002); + nv_wo32(base, 0x84, 0x20400000); + nv_wo32(base, 0x94, 0x30000001); + nv_wo32(base, 0x9c, 0x00000100); + nv_wo32(base, 0xa4, 0x1f1f1f1f); + nv_wo32(base, 0xa8, 0x1f1f1f1f); + nv_wo32(base, 0xac, 0x0000001f); + nv_wo32(base, 0xb8, 0xf8000000); + nv_wo32(base, 0xf8, 0x10003080); /* 0x002310 */ + nv_wo32(base, 0xfc, 0x10000010); /* 0x002350 */ + bar->flush(bar); + return 0; +} + +static int +nvc0_fifo_chan_init(struct nouveau_object *object) +{ + struct nouveau_gpuobj *base = nv_gpuobj(object->parent); + struct nvc0_fifo_priv *priv = (void *)object->engine; + struct nvc0_fifo_chan *chan = (void *)object; + u32 chid = chan->base.chid; + int ret; + + ret = nouveau_fifo_channel_init(&chan->base); + if (ret) + return ret; + + nv_wr32(priv, 0x003000 + (chid * 8), 0xc0000000 | base->addr >> 12); + + if (chan->state == STOPPED && (chan->state = RUNNING) == RUNNING) { + nv_wr32(priv, 0x003004 + (chid * 8), 0x001f0001); + nvc0_fifo_runlist_update(priv); + } + + return 0; +} + +static void nvc0_fifo_intr_engine(struct nvc0_fifo_priv *priv); + +static int +nvc0_fifo_chan_fini(struct nouveau_object *object, bool suspend) +{ + struct nvc0_fifo_priv *priv = (void *)object->engine; + struct nvc0_fifo_chan *chan = (void *)object; + u32 chid = chan->base.chid; + + if (chan->state == RUNNING && (chan->state = STOPPED) == STOPPED) { + nv_mask(priv, 0x003004 + (chid * 8), 0x00000001, 0x00000000); + nvc0_fifo_runlist_update(priv); + } + + nvc0_fifo_intr_engine(priv); + + nv_wr32(priv, 0x003000 + (chid * 8), 0x00000000); + return nouveau_fifo_channel_fini(&chan->base, suspend); +} + +static struct nouveau_ofuncs +nvc0_fifo_ofuncs = { + .ctor = nvc0_fifo_chan_ctor, + .dtor = _nouveau_fifo_channel_dtor, + .init = nvc0_fifo_chan_init, + .fini = nvc0_fifo_chan_fini, + .map = _nouveau_fifo_channel_map, + .rd32 = _nouveau_fifo_channel_rd32, + .wr32 = _nouveau_fifo_channel_wr32, + .ntfy = _nouveau_fifo_channel_ntfy +}; + +static struct nouveau_oclass +nvc0_fifo_sclass[] = { + { FERMI_CHANNEL_GPFIFO, &nvc0_fifo_ofuncs }, + {} +}; + +/******************************************************************************* + * FIFO context - instmem heap and vm setup + ******************************************************************************/ + +static int +nvc0_fifo_context_ctor(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + struct nvc0_fifo_base *base; + int ret; + + ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x1000, + 0x1000, NVOBJ_FLAG_ZERO_ALLOC | + NVOBJ_FLAG_HEAP, &base); + *pobject = nv_object(base); + if (ret) + return ret; + + ret = nouveau_gpuobj_new(nv_object(base), NULL, 0x10000, 0x1000, 0, + &base->pgd); + if (ret) + return ret; + + nv_wo32(base, 0x0200, lower_32_bits(base->pgd->addr)); + nv_wo32(base, 0x0204, upper_32_bits(base->pgd->addr)); + nv_wo32(base, 0x0208, 0xffffffff); + nv_wo32(base, 0x020c, 0x000000ff); + + ret = nouveau_vm_ref(nouveau_client(parent)->vm, &base->vm, base->pgd); + if (ret) + return ret; + + return 0; +} + +static void +nvc0_fifo_context_dtor(struct nouveau_object *object) +{ + struct nvc0_fifo_base *base = (void *)object; + nouveau_vm_ref(NULL, &base->vm, base->pgd); + nouveau_gpuobj_ref(NULL, &base->pgd); + nouveau_fifo_context_destroy(&base->base); +} + +static struct nouveau_oclass +nvc0_fifo_cclass = { + .handle = NV_ENGCTX(FIFO, 0xc0), + .ofuncs = &(struct nouveau_ofuncs) { + .ctor = nvc0_fifo_context_ctor, + .dtor = nvc0_fifo_context_dtor, + .init = _nouveau_fifo_context_init, + .fini = _nouveau_fifo_context_fini, + .rd32 = _nouveau_fifo_context_rd32, + .wr32 = _nouveau_fifo_context_wr32, + }, +}; + +/******************************************************************************* + * PFIFO engine + ******************************************************************************/ + +static inline int +nvc0_fifo_engidx(struct nvc0_fifo_priv *priv, u32 engn) +{ + switch (engn) { + case NVDEV_ENGINE_GR : engn = 0; break; + case NVDEV_ENGINE_BSP : engn = 1; break; + case NVDEV_ENGINE_PPP : engn = 2; break; + case NVDEV_ENGINE_VP : engn = 3; break; + case NVDEV_ENGINE_COPY0: engn = 4; break; + case NVDEV_ENGINE_COPY1: engn = 5; break; + default: + return -1; + } + + return engn; +} + +static inline struct nouveau_engine * +nvc0_fifo_engine(struct nvc0_fifo_priv *priv, u32 engn) +{ + switch (engn) { + case 0: engn = NVDEV_ENGINE_GR; break; + case 1: engn = NVDEV_ENGINE_BSP; break; + case 2: engn = NVDEV_ENGINE_PPP; break; + case 3: engn = NVDEV_ENGINE_VP; break; + case 4: engn = NVDEV_ENGINE_COPY0; break; + case 5: engn = NVDEV_ENGINE_COPY1; break; + default: + return NULL; + } + + return nouveau_engine(priv, engn); +} + +static void +nvc0_fifo_recover_work(struct work_struct *work) +{ + struct nvc0_fifo_priv *priv = container_of(work, typeof(*priv), fault); + struct nouveau_object *engine; + unsigned long flags; + u32 engn, engm = 0; + u64 mask, todo; + + spin_lock_irqsave(&priv->base.lock, flags); + mask = priv->mask; + priv->mask = 0ULL; + spin_unlock_irqrestore(&priv->base.lock, flags); + + for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) + engm |= 1 << nvc0_fifo_engidx(priv, engn); + nv_mask(priv, 0x002630, engm, engm); + + for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) { + if ((engine = (void *)nouveau_engine(priv, engn))) { + nv_ofuncs(engine)->fini(engine, false); + WARN_ON(nv_ofuncs(engine)->init(engine)); + } + } + + nvc0_fifo_runlist_update(priv); + nv_wr32(priv, 0x00262c, engm); + nv_mask(priv, 0x002630, engm, 0x00000000); +} + +static void +nvc0_fifo_recover(struct nvc0_fifo_priv *priv, struct nouveau_engine *engine, + struct nvc0_fifo_chan *chan) +{ + u32 chid = chan->base.chid; + unsigned long flags; + + nv_error(priv, "%s engine fault on channel %d, recovering...\n", + nv_subdev(engine)->name, chid); + + nv_mask(priv, 0x003004 + (chid * 0x08), 0x00000001, 0x00000000); + chan->state = KILLED; + + spin_lock_irqsave(&priv->base.lock, flags); + priv->mask |= 1ULL << nv_engidx(engine); + spin_unlock_irqrestore(&priv->base.lock, flags); + schedule_work(&priv->fault); +} + +static int +nvc0_fifo_swmthd(struct nvc0_fifo_priv *priv, u32 chid, u32 mthd, u32 data) +{ + struct nvc0_fifo_chan *chan = NULL; + struct nouveau_handle *bind; + unsigned long flags; + int ret = -EINVAL; + + spin_lock_irqsave(&priv->base.lock, flags); + if (likely(chid >= priv->base.min && chid <= priv->base.max)) + chan = (void *)priv->base.channel[chid]; + if (unlikely(!chan)) + goto out; + + bind = nouveau_namedb_get_class(nv_namedb(chan), 0x906e); + if (likely(bind)) { + if (!mthd || !nv_call(bind->object, mthd, data)) + ret = 0; + nouveau_namedb_put(bind); + } + +out: + spin_unlock_irqrestore(&priv->base.lock, flags); + return ret; +} + +static const struct nouveau_enum +nvc0_fifo_sched_reason[] = { + { 0x0a, "CTXSW_TIMEOUT" }, + {} +}; + +static void +nvc0_fifo_intr_sched_ctxsw(struct nvc0_fifo_priv *priv) +{ + struct nouveau_engine *engine; + struct nvc0_fifo_chan *chan; + u32 engn; + + for (engn = 0; engn < 6; engn++) { + u32 stat = nv_rd32(priv, 0x002640 + (engn * 0x04)); + u32 busy = (stat & 0x80000000); + u32 save = (stat & 0x00100000); /* maybe? */ + u32 unk0 = (stat & 0x00040000); + u32 unk1 = (stat & 0x00001000); + u32 chid = (stat & 0x0000007f); + (void)save; + + if (busy && unk0 && unk1) { + if (!(chan = (void *)priv->base.channel[chid])) + continue; + if (!(engine = nvc0_fifo_engine(priv, engn))) + continue; + nvc0_fifo_recover(priv, engine, chan); + } + } +} + +static void +nvc0_fifo_intr_sched(struct nvc0_fifo_priv *priv) +{ + u32 intr = nv_rd32(priv, 0x00254c); + u32 code = intr & 0x000000ff; + const struct nouveau_enum *en; + char enunk[6] = ""; + + en = nouveau_enum_find(nvc0_fifo_sched_reason, code); + if (!en) + snprintf(enunk, sizeof(enunk), "UNK%02x", code); + + nv_error(priv, "SCHED_ERROR [ %s ]\n", en ? en->name : enunk); + + switch (code) { + case 0x0a: + nvc0_fifo_intr_sched_ctxsw(priv); + break; + default: + break; + } +} + +static const struct nouveau_enum +nvc0_fifo_fault_engine[] = { + { 0x00, "PGRAPH", NULL, NVDEV_ENGINE_GR }, + { 0x03, "PEEPHOLE", NULL, NVDEV_ENGINE_IFB }, + { 0x04, "BAR1", NULL, NVDEV_SUBDEV_BAR }, + { 0x05, "BAR3", NULL, NVDEV_SUBDEV_INSTMEM }, + { 0x07, "PFIFO", NULL, NVDEV_ENGINE_FIFO }, + { 0x10, "PBSP", NULL, NVDEV_ENGINE_BSP }, + { 0x11, "PPPP", NULL, NVDEV_ENGINE_PPP }, + { 0x13, "PCOUNTER" }, + { 0x14, "PVP", NULL, NVDEV_ENGINE_VP }, + { 0x15, "PCOPY0", NULL, NVDEV_ENGINE_COPY0 }, + { 0x16, "PCOPY1", NULL, NVDEV_ENGINE_COPY1 }, + { 0x17, "PDAEMON" }, + {} +}; + +static const struct nouveau_enum +nvc0_fifo_fault_reason[] = { + { 0x00, "PT_NOT_PRESENT" }, + { 0x01, "PT_TOO_SHORT" }, + { 0x02, "PAGE_NOT_PRESENT" }, + { 0x03, "VM_LIMIT_EXCEEDED" }, + { 0x04, "NO_CHANNEL" }, + { 0x05, "PAGE_SYSTEM_ONLY" }, + { 0x06, "PAGE_READ_ONLY" }, + { 0x0a, "COMPRESSED_SYSRAM" }, + { 0x0c, "INVALID_STORAGE_TYPE" }, + {} +}; + +static const struct nouveau_enum +nvc0_fifo_fault_hubclient[] = { + { 0x01, "PCOPY0" }, + { 0x02, "PCOPY1" }, + { 0x04, "DISPATCH" }, + { 0x05, "CTXCTL" }, + { 0x06, "PFIFO" }, + { 0x07, "BAR_READ" }, + { 0x08, "BAR_WRITE" }, + { 0x0b, "PVP" }, + { 0x0c, "PPPP" }, + { 0x0d, "PBSP" }, + { 0x11, "PCOUNTER" }, + { 0x12, "PDAEMON" }, + { 0x14, "CCACHE" }, + { 0x15, "CCACHE_POST" }, + {} +}; + +static const struct nouveau_enum +nvc0_fifo_fault_gpcclient[] = { + { 0x01, "TEX" }, + { 0x0c, "ESETUP" }, + { 0x0e, "CTXCTL" }, + { 0x0f, "PROP" }, + {} +}; + +static void +nvc0_fifo_intr_fault(struct nvc0_fifo_priv *priv, int unit) +{ + u32 inst = nv_rd32(priv, 0x002800 + (unit * 0x10)); + u32 valo = nv_rd32(priv, 0x002804 + (unit * 0x10)); + u32 vahi = nv_rd32(priv, 0x002808 + (unit * 0x10)); + u32 stat = nv_rd32(priv, 0x00280c + (unit * 0x10)); + u32 gpc = (stat & 0x1f000000) >> 24; + u32 client = (stat & 0x00001f00) >> 8; + u32 write = (stat & 0x00000080); + u32 hub = (stat & 0x00000040); + u32 reason = (stat & 0x0000000f); + struct nouveau_object *engctx = NULL, *object; + struct nouveau_engine *engine = NULL; + const struct nouveau_enum *er, *eu, *ec; + char erunk[6] = ""; + char euunk[6] = ""; + char ecunk[6] = ""; + char gpcid[3] = ""; + + er = nouveau_enum_find(nvc0_fifo_fault_reason, reason); + if (!er) + snprintf(erunk, sizeof(erunk), "UNK%02X", reason); + + eu = nouveau_enum_find(nvc0_fifo_fault_engine, unit); + if (eu) { + switch (eu->data2) { + case NVDEV_SUBDEV_BAR: + nv_mask(priv, 0x001704, 0x00000000, 0x00000000); + break; + case NVDEV_SUBDEV_INSTMEM: + nv_mask(priv, 0x001714, 0x00000000, 0x00000000); + break; + case NVDEV_ENGINE_IFB: + nv_mask(priv, 0x001718, 0x00000000, 0x00000000); + break; + default: + engine = nouveau_engine(priv, eu->data2); + if (engine) + engctx = nouveau_engctx_get(engine, inst); + break; + } + } else { + snprintf(euunk, sizeof(euunk), "UNK%02x", unit); + } + + if (hub) { + ec = nouveau_enum_find(nvc0_fifo_fault_hubclient, client); + } else { + ec = nouveau_enum_find(nvc0_fifo_fault_gpcclient, client); + snprintf(gpcid, sizeof(gpcid), "%d", gpc); + } + + if (!ec) + snprintf(ecunk, sizeof(ecunk), "UNK%02x", client); + + nv_error(priv, "%s fault at 0x%010llx [%s] from %s/%s%s%s%s on " + "channel 0x%010llx [%s]\n", write ? "write" : "read", + (u64)vahi << 32 | valo, er ? er->name : erunk, + eu ? eu->name : euunk, hub ? "" : "GPC", gpcid, hub ? "" : "/", + ec ? ec->name : ecunk, (u64)inst << 12, + nouveau_client_name(engctx)); + + object = engctx; + while (object) { + switch (nv_mclass(object)) { + case FERMI_CHANNEL_GPFIFO: + nvc0_fifo_recover(priv, engine, (void *)object); + break; + } + object = object->parent; + } + + nouveau_engctx_put(engctx); +} + +static const struct nouveau_bitfield +nvc0_fifo_pbdma_intr[] = { +/* { 0x00008000, "" } seen with null ib push */ + { 0x00200000, "ILLEGAL_MTHD" }, + { 0x00800000, "EMPTY_SUBC" }, + {} +}; + +static void +nvc0_fifo_intr_pbdma(struct nvc0_fifo_priv *priv, int unit) +{ + u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000)); + u32 addr = nv_rd32(priv, 0x0400c0 + (unit * 0x2000)); + u32 data = nv_rd32(priv, 0x0400c4 + (unit * 0x2000)); + u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0x7f; + u32 subc = (addr & 0x00070000) >> 16; + u32 mthd = (addr & 0x00003ffc); + u32 show = stat; + + if (stat & 0x00800000) { + if (!nvc0_fifo_swmthd(priv, chid, mthd, data)) + show &= ~0x00800000; + } + + if (show) { + nv_error(priv, "PBDMA%d:", unit); + nouveau_bitfield_print(nvc0_fifo_pbdma_intr, show); + pr_cont("\n"); + nv_error(priv, + "PBDMA%d: ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n", + unit, chid, + nouveau_client_name_for_fifo_chid(&priv->base, chid), + subc, mthd, data); + } + + nv_wr32(priv, 0x0400c0 + (unit * 0x2000), 0x80600008); + nv_wr32(priv, 0x040108 + (unit * 0x2000), stat); +} + +static void +nvc0_fifo_intr_runlist(struct nvc0_fifo_priv *priv) +{ + u32 intr = nv_rd32(priv, 0x002a00); + + if (intr & 0x10000000) { + wake_up(&priv->runlist.wait); + nv_wr32(priv, 0x002a00, 0x10000000); + intr &= ~0x10000000; + } + + if (intr) { + nv_error(priv, "RUNLIST 0x%08x\n", intr); + nv_wr32(priv, 0x002a00, intr); + } +} + +static void +nvc0_fifo_intr_engine_unit(struct nvc0_fifo_priv *priv, int engn) +{ + u32 intr = nv_rd32(priv, 0x0025a8 + (engn * 0x04)); + u32 inte = nv_rd32(priv, 0x002628); + u32 unkn; + + nv_wr32(priv, 0x0025a8 + (engn * 0x04), intr); + + for (unkn = 0; unkn < 8; unkn++) { + u32 ints = (intr >> (unkn * 0x04)) & inte; + if (ints & 0x1) { + nouveau_fifo_uevent(&priv->base); + ints &= ~1; + } + if (ints) { + nv_error(priv, "ENGINE %d %d %01x", engn, unkn, ints); + nv_mask(priv, 0x002628, ints, 0); + } + } +} + +static void +nvc0_fifo_intr_engine(struct nvc0_fifo_priv *priv) +{ + u32 mask = nv_rd32(priv, 0x0025a4); + while (mask) { + u32 unit = __ffs(mask); + nvc0_fifo_intr_engine_unit(priv, unit); + mask &= ~(1 << unit); + } +} + +static void +nvc0_fifo_intr(struct nouveau_subdev *subdev) +{ + struct nvc0_fifo_priv *priv = (void *)subdev; + u32 mask = nv_rd32(priv, 0x002140); + u32 stat = nv_rd32(priv, 0x002100) & mask; + + if (stat & 0x00000001) { + u32 intr = nv_rd32(priv, 0x00252c); + nv_warn(priv, "INTR 0x00000001: 0x%08x\n", intr); + nv_wr32(priv, 0x002100, 0x00000001); + stat &= ~0x00000001; + } + + if (stat & 0x00000100) { + nvc0_fifo_intr_sched(priv); + nv_wr32(priv, 0x002100, 0x00000100); + stat &= ~0x00000100; + } + + if (stat & 0x00010000) { + u32 intr = nv_rd32(priv, 0x00256c); + nv_warn(priv, "INTR 0x00010000: 0x%08x\n", intr); + nv_wr32(priv, 0x002100, 0x00010000); + stat &= ~0x00010000; + } + + if (stat & 0x01000000) { + u32 intr = nv_rd32(priv, 0x00258c); + nv_warn(priv, "INTR 0x01000000: 0x%08x\n", intr); + nv_wr32(priv, 0x002100, 0x01000000); + stat &= ~0x01000000; + } + + if (stat & 0x10000000) { + u32 mask = nv_rd32(priv, 0x00259c); + while (mask) { + u32 unit = __ffs(mask); + nvc0_fifo_intr_fault(priv, unit); + nv_wr32(priv, 0x00259c, (1 << unit)); + mask &= ~(1 << unit); + } + stat &= ~0x10000000; + } + + if (stat & 0x20000000) { + u32 mask = nv_rd32(priv, 0x0025a0); + while (mask) { + u32 unit = __ffs(mask); + nvc0_fifo_intr_pbdma(priv, unit); + nv_wr32(priv, 0x0025a0, (1 << unit)); + mask &= ~(1 << unit); + } + stat &= ~0x20000000; + } + + if (stat & 0x40000000) { + nvc0_fifo_intr_runlist(priv); + stat &= ~0x40000000; + } + + if (stat & 0x80000000) { + nvc0_fifo_intr_engine(priv); + stat &= ~0x80000000; + } + + if (stat) { + nv_error(priv, "INTR 0x%08x\n", stat); + nv_mask(priv, 0x002140, stat, 0x00000000); + nv_wr32(priv, 0x002100, stat); + } +} + +static void +nvc0_fifo_uevent_init(struct nvkm_event *event, int type, int index) +{ + struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent); + nv_mask(fifo, 0x002140, 0x80000000, 0x80000000); +} + +static void +nvc0_fifo_uevent_fini(struct nvkm_event *event, int type, int index) +{ + struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent); + nv_mask(fifo, 0x002140, 0x80000000, 0x00000000); +} + +static const struct nvkm_event_func +nvc0_fifo_uevent_func = { + .ctor = nouveau_fifo_uevent_ctor, + .init = nvc0_fifo_uevent_init, + .fini = nvc0_fifo_uevent_fini, +}; + +static int +nvc0_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + struct nvc0_fifo_priv *priv; + int ret; + + ret = nouveau_fifo_create(parent, engine, oclass, 0, 127, &priv); + *pobject = nv_object(priv); + if (ret) + return ret; + + INIT_WORK(&priv->fault, nvc0_fifo_recover_work); + + ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x1000, 0x1000, 0, + &priv->runlist.mem[0]); + if (ret) + return ret; + + ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x1000, 0x1000, 0, + &priv->runlist.mem[1]); + if (ret) + return ret; + + init_waitqueue_head(&priv->runlist.wait); + + ret = nouveau_gpuobj_new(nv_object(priv), NULL, 128 * 0x1000, 0x1000, 0, + &priv->user.mem); + if (ret) + return ret; + + ret = nouveau_gpuobj_map(priv->user.mem, NV_MEM_ACCESS_RW, + &priv->user.bar); + if (ret) + return ret; + + ret = nvkm_event_init(&nvc0_fifo_uevent_func, 1, 1, &priv->base.uevent); + if (ret) + return ret; + + nv_subdev(priv)->unit = 0x00000100; + nv_subdev(priv)->intr = nvc0_fifo_intr; + nv_engine(priv)->cclass = &nvc0_fifo_cclass; + nv_engine(priv)->sclass = nvc0_fifo_sclass; + return 0; +} + +static void +nvc0_fifo_dtor(struct nouveau_object *object) +{ + struct nvc0_fifo_priv *priv = (void *)object; + + nouveau_gpuobj_unmap(&priv->user.bar); + nouveau_gpuobj_ref(NULL, &priv->user.mem); + nouveau_gpuobj_ref(NULL, &priv->runlist.mem[0]); + nouveau_gpuobj_ref(NULL, &priv->runlist.mem[1]); + + nouveau_fifo_destroy(&priv->base); +} + +static int +nvc0_fifo_init(struct nouveau_object *object) +{ + struct nvc0_fifo_priv *priv = (void *)object; + int ret, i; + + ret = nouveau_fifo_init(&priv->base); + if (ret) + return ret; + + nv_wr32(priv, 0x000204, 0xffffffff); + nv_wr32(priv, 0x002204, 0xffffffff); + + priv->spoon_nr = hweight32(nv_rd32(priv, 0x002204)); + nv_debug(priv, "%d PBDMA unit(s)\n", priv->spoon_nr); + + /* assign engines to PBDMAs */ + if (priv->spoon_nr >= 3) { + nv_wr32(priv, 0x002208, ~(1 << 0)); /* PGRAPH */ + nv_wr32(priv, 0x00220c, ~(1 << 1)); /* PVP */ + nv_wr32(priv, 0x002210, ~(1 << 1)); /* PPP */ + nv_wr32(priv, 0x002214, ~(1 << 1)); /* PBSP */ + nv_wr32(priv, 0x002218, ~(1 << 2)); /* PCE0 */ + nv_wr32(priv, 0x00221c, ~(1 << 1)); /* PCE1 */ + } + + /* PBDMA[n] */ + for (i = 0; i < priv->spoon_nr; i++) { + nv_mask(priv, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000); + nv_wr32(priv, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */ + nv_wr32(priv, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */ + } + + nv_mask(priv, 0x002200, 0x00000001, 0x00000001); + nv_wr32(priv, 0x002254, 0x10000000 | priv->user.bar.offset >> 12); + + nv_wr32(priv, 0x002100, 0xffffffff); + nv_wr32(priv, 0x002140, 0x7fffffff); + nv_wr32(priv, 0x002628, 0x00000001); /* ENGINE_INTR_EN */ + return 0; +} + +struct nouveau_oclass * +nvc0_fifo_oclass = &(struct nouveau_oclass) { + .handle = NV_ENGINE(FIFO, 0xc0), + .ofuncs = &(struct nouveau_ofuncs) { + .ctor = nvc0_fifo_ctor, + .dtor = nvc0_fifo_dtor, + .init = nvc0_fifo_init, + .fini = _nouveau_fifo_fini, + }, +}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nve0.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nve0.c new file mode 100644 index 000000000000..c9c168035b4d --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nve0.c @@ -0,0 +1,1146 @@ +/* + * Copyright 2012 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * Authors: Ben Skeggs + */ + +#include <core/client.h> +#include <core/handle.h> +#include <core/namedb.h> +#include <core/gpuobj.h> +#include <core/engctx.h> +#include <core/event.h> +#include <nvif/unpack.h> +#include <nvif/class.h> +#include <core/enum.h> + +#include <subdev/timer.h> +#include <subdev/bar.h> +#include <subdev/fb.h> +#include <subdev/vm.h> + +#include <engine/dmaobj.h> + +#include "nve0.h" + +#define _(a,b) { (a), ((1ULL << (a)) | (b)) } +static const struct { + u64 subdev; + u64 mask; +} fifo_engine[] = { + _(NVDEV_ENGINE_GR , (1ULL << NVDEV_ENGINE_SW) | + (1ULL << NVDEV_ENGINE_COPY2)), + _(NVDEV_ENGINE_VP , 0), + _(NVDEV_ENGINE_PPP , 0), + _(NVDEV_ENGINE_BSP , 0), + _(NVDEV_ENGINE_COPY0 , 0), + _(NVDEV_ENGINE_COPY1 , 0), + _(NVDEV_ENGINE_VENC , 0), +}; +#undef _ +#define FIFO_ENGINE_NR ARRAY_SIZE(fifo_engine) + +struct nve0_fifo_engn { + struct nouveau_gpuobj *runlist[2]; + int cur_runlist; + wait_queue_head_t wait; +}; + +struct nve0_fifo_priv { + struct nouveau_fifo base; + + struct work_struct fault; + u64 mask; + + struct nve0_fifo_engn engine[FIFO_ENGINE_NR]; + struct { + struct nouveau_gpuobj *mem; + struct nouveau_vma bar; + } user; + int spoon_nr; +}; + +struct nve0_fifo_base { + struct nouveau_fifo_base base; + struct nouveau_gpuobj *pgd; + struct nouveau_vm *vm; +}; + +struct nve0_fifo_chan { + struct nouveau_fifo_chan base; + u32 engine; + enum { + STOPPED, + RUNNING, + KILLED + } state; +}; + +/******************************************************************************* + * FIFO channel objects + ******************************************************************************/ + +static void +nve0_fifo_runlist_update(struct nve0_fifo_priv *priv, u32 engine) +{ + struct nouveau_bar *bar = nouveau_bar(priv); + struct nve0_fifo_engn *engn = &priv->engine[engine]; + struct nouveau_gpuobj *cur; + int i, p; + + mutex_lock(&nv_subdev(priv)->mutex); + cur = engn->runlist[engn->cur_runlist]; + engn->cur_runlist = !engn->cur_runlist; + + for (i = 0, p = 0; i < priv->base.max; i++) { + struct nve0_fifo_chan *chan = (void *)priv->base.channel[i]; + if (chan && chan->state == RUNNING && chan->engine == engine) { + nv_wo32(cur, p + 0, i); + nv_wo32(cur, p + 4, 0x00000000); + p += 8; + } + } + bar->flush(bar); + + nv_wr32(priv, 0x002270, cur->addr >> 12); + nv_wr32(priv, 0x002274, (engine << 20) | (p >> 3)); + + if (wait_event_timeout(engn->wait, !(nv_rd32(priv, 0x002284 + + (engine * 0x08)) & 0x00100000), + msecs_to_jiffies(2000)) == 0) + nv_error(priv, "runlist %d update timeout\n", engine); + mutex_unlock(&nv_subdev(priv)->mutex); +} + +static int +nve0_fifo_context_attach(struct nouveau_object *parent, + struct nouveau_object *object) +{ + struct nouveau_bar *bar = nouveau_bar(parent); + struct nve0_fifo_base *base = (void *)parent->parent; + struct nouveau_engctx *ectx = (void *)object; + u32 addr; + int ret; + + switch (nv_engidx(object->engine)) { + case NVDEV_ENGINE_SW : + return 0; + case NVDEV_ENGINE_COPY0: + case NVDEV_ENGINE_COPY1: + case NVDEV_ENGINE_COPY2: + nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12; + return 0; + case NVDEV_ENGINE_GR : addr = 0x0210; break; + case NVDEV_ENGINE_BSP : addr = 0x0270; break; + case NVDEV_ENGINE_VP : addr = 0x0250; break; + case NVDEV_ENGINE_PPP : addr = 0x0260; break; + default: + return -EINVAL; + } + + if (!ectx->vma.node) { + ret = nouveau_gpuobj_map_vm(nv_gpuobj(ectx), base->vm, + NV_MEM_ACCESS_RW, &ectx->vma); + if (ret) + return ret; + + nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12; + } + + nv_wo32(base, addr + 0x00, lower_32_bits(ectx->vma.offset) | 4); + nv_wo32(base, addr + 0x04, upper_32_bits(ectx->vma.offset)); + bar->flush(bar); + return 0; +} + +static int +nve0_fifo_context_detach(struct nouveau_object *parent, bool suspend, + struct nouveau_object *object) +{ + struct nouveau_bar *bar = nouveau_bar(parent); + struct nve0_fifo_priv *priv = (void *)parent->engine; + struct nve0_fifo_base *base = (void *)parent->parent; + struct nve0_fifo_chan *chan = (void *)parent; + u32 addr; + + switch (nv_engidx(object->engine)) { + case NVDEV_ENGINE_SW : return 0; + case NVDEV_ENGINE_COPY0: + case NVDEV_ENGINE_COPY1: + case NVDEV_ENGINE_COPY2: addr = 0x0000; break; + case NVDEV_ENGINE_GR : addr = 0x0210; break; + case NVDEV_ENGINE_BSP : addr = 0x0270; break; + case NVDEV_ENGINE_VP : addr = 0x0250; break; + case NVDEV_ENGINE_PPP : addr = 0x0260; break; + default: + return -EINVAL; + } + + nv_wr32(priv, 0x002634, chan->base.chid); + if (!nv_wait(priv, 0x002634, 0xffffffff, chan->base.chid)) { + nv_error(priv, "channel %d [%s] kick timeout\n", + chan->base.chid, nouveau_client_name(chan)); + if (suspend) + return -EBUSY; + } + + if (addr) { + nv_wo32(base, addr + 0x00, 0x00000000); + nv_wo32(base, addr + 0x04, 0x00000000); + bar->flush(bar); + } + + return 0; +} + +static int +nve0_fifo_chan_ctor(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + union { + struct kepler_channel_gpfifo_a_v0 v0; + } *args = data; + struct nouveau_bar *bar = nouveau_bar(parent); + struct nve0_fifo_priv *priv = (void *)engine; + struct nve0_fifo_base *base = (void *)parent; + struct nve0_fifo_chan *chan; + u64 usermem, ioffset, ilength; + int ret, i; + + nv_ioctl(parent, "create channel gpfifo size %d\n", size); + if (nvif_unpack(args->v0, 0, 0, false)) { + nv_ioctl(parent, "create channel gpfifo vers %d pushbuf %08x " + "ioffset %016llx ilength %08x engine %08x\n", + args->v0.version, args->v0.pushbuf, args->v0.ioffset, + args->v0.ilength, args->v0.engine); + } else + return ret; + + for (i = 0; i < FIFO_ENGINE_NR; i++) { + if (args->v0.engine & (1 << i)) { + if (nouveau_engine(parent, fifo_engine[i].subdev)) { + args->v0.engine = (1 << i); + break; + } + } + } + + if (i == FIFO_ENGINE_NR) { + nv_error(priv, "unsupported engines 0x%08x\n", args->v0.engine); + return -ENODEV; + } + + ret = nouveau_fifo_channel_create(parent, engine, oclass, 1, + priv->user.bar.offset, 0x200, + args->v0.pushbuf, + fifo_engine[i].mask, &chan); + *pobject = nv_object(chan); + if (ret) + return ret; + + args->v0.chid = chan->base.chid; + + nv_parent(chan)->context_attach = nve0_fifo_context_attach; + nv_parent(chan)->context_detach = nve0_fifo_context_detach; + chan->engine = i; + + usermem = chan->base.chid * 0x200; + ioffset = args->v0.ioffset; + ilength = order_base_2(args->v0.ilength / 8); + + for (i = 0; i < 0x200; i += 4) + nv_wo32(priv->user.mem, usermem + i, 0x00000000); + + nv_wo32(base, 0x08, lower_32_bits(priv->user.mem->addr + usermem)); + nv_wo32(base, 0x0c, upper_32_bits(priv->user.mem->addr + usermem)); + nv_wo32(base, 0x10, 0x0000face); + nv_wo32(base, 0x30, 0xfffff902); + nv_wo32(base, 0x48, lower_32_bits(ioffset)); + nv_wo32(base, 0x4c, upper_32_bits(ioffset) | (ilength << 16)); + nv_wo32(base, 0x84, 0x20400000); + nv_wo32(base, 0x94, 0x30000001); + nv_wo32(base, 0x9c, 0x00000100); + nv_wo32(base, 0xac, 0x0000001f); + nv_wo32(base, 0xe8, chan->base.chid); + nv_wo32(base, 0xb8, 0xf8000000); + nv_wo32(base, 0xf8, 0x10003080); /* 0x002310 */ + nv_wo32(base, 0xfc, 0x10000010); /* 0x002350 */ + bar->flush(bar); + return 0; +} + +static int +nve0_fifo_chan_init(struct nouveau_object *object) +{ + struct nouveau_gpuobj *base = nv_gpuobj(object->parent); + struct nve0_fifo_priv *priv = (void *)object->engine; + struct nve0_fifo_chan *chan = (void *)object; + u32 chid = chan->base.chid; + int ret; + + ret = nouveau_fifo_channel_init(&chan->base); + if (ret) + return ret; + + nv_mask(priv, 0x800004 + (chid * 8), 0x000f0000, chan->engine << 16); + nv_wr32(priv, 0x800000 + (chid * 8), 0x80000000 | base->addr >> 12); + + if (chan->state == STOPPED && (chan->state = RUNNING) == RUNNING) { + nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400); + nve0_fifo_runlist_update(priv, chan->engine); + nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400); + } + + return 0; +} + +static int +nve0_fifo_chan_fini(struct nouveau_object *object, bool suspend) +{ + struct nve0_fifo_priv *priv = (void *)object->engine; + struct nve0_fifo_chan *chan = (void *)object; + u32 chid = chan->base.chid; + + if (chan->state == RUNNING && (chan->state = STOPPED) == STOPPED) { + nv_mask(priv, 0x800004 + (chid * 8), 0x00000800, 0x00000800); + nve0_fifo_runlist_update(priv, chan->engine); + } + + nv_wr32(priv, 0x800000 + (chid * 8), 0x00000000); + return nouveau_fifo_channel_fini(&chan->base, suspend); +} + +static struct nouveau_ofuncs +nve0_fifo_ofuncs = { + .ctor = nve0_fifo_chan_ctor, + .dtor = _nouveau_fifo_channel_dtor, + .init = nve0_fifo_chan_init, + .fini = nve0_fifo_chan_fini, + .map = _nouveau_fifo_channel_map, + .rd32 = _nouveau_fifo_channel_rd32, + .wr32 = _nouveau_fifo_channel_wr32, + .ntfy = _nouveau_fifo_channel_ntfy +}; + +static struct nouveau_oclass +nve0_fifo_sclass[] = { + { KEPLER_CHANNEL_GPFIFO_A, &nve0_fifo_ofuncs }, + {} +}; + +/******************************************************************************* + * FIFO context - instmem heap and vm setup + ******************************************************************************/ + +static int +nve0_fifo_context_ctor(struct nouveau_object *parent, + struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + struct nve0_fifo_base *base; + int ret; + + ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x1000, + 0x1000, NVOBJ_FLAG_ZERO_ALLOC, &base); + *pobject = nv_object(base); + if (ret) + return ret; + + ret = nouveau_gpuobj_new(nv_object(base), NULL, 0x10000, 0x1000, 0, + &base->pgd); + if (ret) + return ret; + + nv_wo32(base, 0x0200, lower_32_bits(base->pgd->addr)); + nv_wo32(base, 0x0204, upper_32_bits(base->pgd->addr)); + nv_wo32(base, 0x0208, 0xffffffff); + nv_wo32(base, 0x020c, 0x000000ff); + + ret = nouveau_vm_ref(nouveau_client(parent)->vm, &base->vm, base->pgd); + if (ret) + return ret; + + return 0; +} + +static void +nve0_fifo_context_dtor(struct nouveau_object *object) +{ + struct nve0_fifo_base *base = (void *)object; + nouveau_vm_ref(NULL, &base->vm, base->pgd); + nouveau_gpuobj_ref(NULL, &base->pgd); + nouveau_fifo_context_destroy(&base->base); +} + +static struct nouveau_oclass +nve0_fifo_cclass = { + .handle = NV_ENGCTX(FIFO, 0xe0), + .ofuncs = &(struct nouveau_ofuncs) { + .ctor = nve0_fifo_context_ctor, + .dtor = nve0_fifo_context_dtor, + .init = _nouveau_fifo_context_init, + .fini = _nouveau_fifo_context_fini, + .rd32 = _nouveau_fifo_context_rd32, + .wr32 = _nouveau_fifo_context_wr32, + }, +}; + +/******************************************************************************* + * PFIFO engine + ******************************************************************************/ + +static inline int +nve0_fifo_engidx(struct nve0_fifo_priv *priv, u32 engn) +{ + switch (engn) { + case NVDEV_ENGINE_GR : + case NVDEV_ENGINE_COPY2: engn = 0; break; + case NVDEV_ENGINE_BSP : engn = 1; break; + case NVDEV_ENGINE_PPP : engn = 2; break; + case NVDEV_ENGINE_VP : engn = 3; break; + case NVDEV_ENGINE_COPY0: engn = 4; break; + case NVDEV_ENGINE_COPY1: engn = 5; break; + case NVDEV_ENGINE_VENC : engn = 6; break; + default: + return -1; + } + + return engn; +} + +static inline struct nouveau_engine * +nve0_fifo_engine(struct nve0_fifo_priv *priv, u32 engn) +{ + if (engn >= ARRAY_SIZE(fifo_engine)) + return NULL; + return nouveau_engine(priv, fifo_engine[engn].subdev); +} + +static void +nve0_fifo_recover_work(struct work_struct *work) +{ + struct nve0_fifo_priv *priv = container_of(work, typeof(*priv), fault); + struct nouveau_object *engine; + unsigned long flags; + u32 engn, engm = 0; + u64 mask, todo; + + spin_lock_irqsave(&priv->base.lock, flags); + mask = priv->mask; + priv->mask = 0ULL; + spin_unlock_irqrestore(&priv->base.lock, flags); + + for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) + engm |= 1 << nve0_fifo_engidx(priv, engn); + nv_mask(priv, 0x002630, engm, engm); + + for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) { + if ((engine = (void *)nouveau_engine(priv, engn))) { + nv_ofuncs(engine)->fini(engine, false); + WARN_ON(nv_ofuncs(engine)->init(engine)); + } + nve0_fifo_runlist_update(priv, nve0_fifo_engidx(priv, engn)); + } + + nv_wr32(priv, 0x00262c, engm); + nv_mask(priv, 0x002630, engm, 0x00000000); +} + +static void +nve0_fifo_recover(struct nve0_fifo_priv *priv, struct nouveau_engine *engine, + struct nve0_fifo_chan *chan) +{ + u32 chid = chan->base.chid; + unsigned long flags; + + nv_error(priv, "%s engine fault on channel %d, recovering...\n", + nv_subdev(engine)->name, chid); + + nv_mask(priv, 0x800004 + (chid * 0x08), 0x00000800, 0x00000800); + chan->state = KILLED; + + spin_lock_irqsave(&priv->base.lock, flags); + priv->mask |= 1ULL << nv_engidx(engine); + spin_unlock_irqrestore(&priv->base.lock, flags); + schedule_work(&priv->fault); +} + +static int +nve0_fifo_swmthd(struct nve0_fifo_priv *priv, u32 chid, u32 mthd, u32 data) +{ + struct nve0_fifo_chan *chan = NULL; + struct nouveau_handle *bind; + unsigned long flags; + int ret = -EINVAL; + + spin_lock_irqsave(&priv->base.lock, flags); + if (likely(chid >= priv->base.min && chid <= priv->base.max)) + chan = (void *)priv->base.channel[chid]; + if (unlikely(!chan)) + goto out; + + bind = nouveau_namedb_get_class(nv_namedb(chan), 0x906e); + if (likely(bind)) { + if (!mthd || !nv_call(bind->object, mthd, data)) + ret = 0; + nouveau_namedb_put(bind); + } + +out: + spin_unlock_irqrestore(&priv->base.lock, flags); + return ret; +} + +static const struct nouveau_enum +nve0_fifo_bind_reason[] = { + { 0x01, "BIND_NOT_UNBOUND" }, + { 0x02, "SNOOP_WITHOUT_BAR1" }, + { 0x03, "UNBIND_WHILE_RUNNING" }, + { 0x05, "INVALID_RUNLIST" }, + { 0x06, "INVALID_CTX_TGT" }, + { 0x0b, "UNBIND_WHILE_PARKED" }, + {} +}; + +static void +nve0_fifo_intr_bind(struct nve0_fifo_priv *priv) +{ + u32 intr = nv_rd32(priv, 0x00252c); + u32 code = intr & 0x000000ff; + const struct nouveau_enum *en; + char enunk[6] = ""; + + en = nouveau_enum_find(nve0_fifo_bind_reason, code); + if (!en) + snprintf(enunk, sizeof(enunk), "UNK%02x", code); + + nv_error(priv, "BIND_ERROR [ %s ]\n", en ? en->name : enunk); +} + +static const struct nouveau_enum +nve0_fifo_sched_reason[] = { + { 0x0a, "CTXSW_TIMEOUT" }, + {} +}; + +static void +nve0_fifo_intr_sched_ctxsw(struct nve0_fifo_priv *priv) +{ + struct nouveau_engine *engine; + struct nve0_fifo_chan *chan; + u32 engn; + + for (engn = 0; engn < ARRAY_SIZE(fifo_engine); engn++) { + u32 stat = nv_rd32(priv, 0x002640 + (engn * 0x04)); + u32 busy = (stat & 0x80000000); + u32 next = (stat & 0x07ff0000) >> 16; + u32 chsw = (stat & 0x00008000); + u32 save = (stat & 0x00004000); + u32 load = (stat & 0x00002000); + u32 prev = (stat & 0x000007ff); + u32 chid = load ? next : prev; + (void)save; + + if (busy && chsw) { + if (!(chan = (void *)priv->base.channel[chid])) + continue; + if (!(engine = nve0_fifo_engine(priv, engn))) + continue; + nve0_fifo_recover(priv, engine, chan); + } + } +} + +static void +nve0_fifo_intr_sched(struct nve0_fifo_priv *priv) +{ + u32 intr = nv_rd32(priv, 0x00254c); + u32 code = intr & 0x000000ff; + const struct nouveau_enum *en; + char enunk[6] = ""; + + en = nouveau_enum_find(nve0_fifo_sched_reason, code); + if (!en) + snprintf(enunk, sizeof(enunk), "UNK%02x", code); + + nv_error(priv, "SCHED_ERROR [ %s ]\n", en ? en->name : enunk); + + switch (code) { + case 0x0a: + nve0_fifo_intr_sched_ctxsw(priv); + break; + default: + break; + } +} + +static void +nve0_fifo_intr_chsw(struct nve0_fifo_priv *priv) +{ + u32 stat = nv_rd32(priv, 0x00256c); + nv_error(priv, "CHSW_ERROR 0x%08x\n", stat); + nv_wr32(priv, 0x00256c, stat); +} + +static void +nve0_fifo_intr_dropped_fault(struct nve0_fifo_priv *priv) +{ + u32 stat = nv_rd32(priv, 0x00259c); + nv_error(priv, "DROPPED_MMU_FAULT 0x%08x\n", stat); +} + +static const struct nouveau_enum +nve0_fifo_fault_engine[] = { + { 0x00, "GR", NULL, NVDEV_ENGINE_GR }, + { 0x03, "IFB", NULL, NVDEV_ENGINE_IFB }, + { 0x04, "BAR1", NULL, NVDEV_SUBDEV_BAR }, + { 0x05, "BAR3", NULL, NVDEV_SUBDEV_INSTMEM }, + { 0x07, "PBDMA0", NULL, NVDEV_ENGINE_FIFO }, + { 0x08, "PBDMA1", NULL, NVDEV_ENGINE_FIFO }, + { 0x09, "PBDMA2", NULL, NVDEV_ENGINE_FIFO }, + { 0x10, "MSVLD", NULL, NVDEV_ENGINE_BSP }, + { 0x11, "MSPPP", NULL, NVDEV_ENGINE_PPP }, + { 0x13, "PERF" }, + { 0x14, "MSPDEC", NULL, NVDEV_ENGINE_VP }, + { 0x15, "CE0", NULL, NVDEV_ENGINE_COPY0 }, + { 0x16, "CE1", NULL, NVDEV_ENGINE_COPY1 }, + { 0x17, "PMU" }, + { 0x19, "MSENC", NULL, NVDEV_ENGINE_VENC }, + { 0x1b, "CE2", NULL, NVDEV_ENGINE_COPY2 }, + {} +}; + +static const struct nouveau_enum +nve0_fifo_fault_reason[] = { + { 0x00, "PDE" }, + { 0x01, "PDE_SIZE" }, + { 0x02, "PTE" }, + { 0x03, "VA_LIMIT_VIOLATION" }, + { 0x04, "UNBOUND_INST_BLOCK" }, + { 0x05, "PRIV_VIOLATION" }, + { 0x06, "RO_VIOLATION" }, + { 0x07, "WO_VIOLATION" }, + { 0x08, "PITCH_MASK_VIOLATION" }, + { 0x09, "WORK_CREATION" }, + { 0x0a, "UNSUPPORTED_APERTURE" }, + { 0x0b, "COMPRESSION_FAILURE" }, + { 0x0c, "UNSUPPORTED_KIND" }, + { 0x0d, "REGION_VIOLATION" }, + { 0x0e, "BOTH_PTES_VALID" }, + { 0x0f, "INFO_TYPE_POISONED" }, + {} +}; + +static const struct nouveau_enum +nve0_fifo_fault_hubclient[] = { + { 0x00, "VIP" }, + { 0x01, "CE0" }, + { 0x02, "CE1" }, + { 0x03, "DNISO" }, + { 0x04, "FE" }, + { 0x05, "FECS" }, + { 0x06, "HOST" }, + { 0x07, "HOST_CPU" }, + { 0x08, "HOST_CPU_NB" }, + { 0x09, "ISO" }, + { 0x0a, "MMU" }, + { 0x0b, "MSPDEC" }, + { 0x0c, "MSPPP" }, + { 0x0d, "MSVLD" }, + { 0x0e, "NISO" }, + { 0x0f, "P2P" }, + { 0x10, "PD" }, + { 0x11, "PERF" }, + { 0x12, "PMU" }, + { 0x13, "RASTERTWOD" }, + { 0x14, "SCC" }, + { 0x15, "SCC_NB" }, + { 0x16, "SEC" }, + { 0x17, "SSYNC" }, + { 0x18, "GR_COPY" }, + { 0x19, "CE2" }, + { 0x1a, "XV" }, + { 0x1b, "MMU_NB" }, + { 0x1c, "MSENC" }, + { 0x1d, "DFALCON" }, + { 0x1e, "SKED" }, + { 0x1f, "AFALCON" }, + {} +}; + +static const struct nouveau_enum +nve0_fifo_fault_gpcclient[] = { + { 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" }, + { 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" }, + { 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" }, + { 0x09, "L1_3" }, { 0x0a, "T1_3" }, { 0x0b, "PE_3" }, + { 0x0c, "RAST" }, + { 0x0d, "GCC" }, + { 0x0e, "GPCCS" }, + { 0x0f, "PROP_0" }, + { 0x10, "PROP_1" }, + { 0x11, "PROP_2" }, + { 0x12, "PROP_3" }, + { 0x13, "L1_4" }, { 0x14, "T1_4" }, { 0x15, "PE_4" }, + { 0x16, "L1_5" }, { 0x17, "T1_5" }, { 0x18, "PE_5" }, + { 0x19, "L1_6" }, { 0x1a, "T1_6" }, { 0x1b, "PE_6" }, + { 0x1c, "L1_7" }, { 0x1d, "T1_7" }, { 0x1e, "PE_7" }, + { 0x1f, "GPM" }, + { 0x20, "LTP_UTLB_0" }, + { 0x21, "LTP_UTLB_1" }, + { 0x22, "LTP_UTLB_2" }, + { 0x23, "LTP_UTLB_3" }, + { 0x24, "GPC_RGG_UTLB" }, + {} +}; + +static void +nve0_fifo_intr_fault(struct nve0_fifo_priv *priv, int unit) +{ + u32 inst = nv_rd32(priv, 0x002800 + (unit * 0x10)); + u32 valo = nv_rd32(priv, 0x002804 + (unit * 0x10)); + u32 vahi = nv_rd32(priv, 0x002808 + (unit * 0x10)); + u32 stat = nv_rd32(priv, 0x00280c + (unit * 0x10)); + u32 gpc = (stat & 0x1f000000) >> 24; + u32 client = (stat & 0x00001f00) >> 8; + u32 write = (stat & 0x00000080); + u32 hub = (stat & 0x00000040); + u32 reason = (stat & 0x0000000f); + struct nouveau_object *engctx = NULL, *object; + struct nouveau_engine *engine = NULL; + const struct nouveau_enum *er, *eu, *ec; + char erunk[6] = ""; + char euunk[6] = ""; + char ecunk[6] = ""; + char gpcid[3] = ""; + + er = nouveau_enum_find(nve0_fifo_fault_reason, reason); + if (!er) + snprintf(erunk, sizeof(erunk), "UNK%02X", reason); + + eu = nouveau_enum_find(nve0_fifo_fault_engine, unit); + if (eu) { + switch (eu->data2) { + case NVDEV_SUBDEV_BAR: + nv_mask(priv, 0x001704, 0x00000000, 0x00000000); + break; + case NVDEV_SUBDEV_INSTMEM: + nv_mask(priv, 0x001714, 0x00000000, 0x00000000); + break; + case NVDEV_ENGINE_IFB: + nv_mask(priv, 0x001718, 0x00000000, 0x00000000); + break; + default: + engine = nouveau_engine(priv, eu->data2); + if (engine) + engctx = nouveau_engctx_get(engine, inst); + break; + } + } else { + snprintf(euunk, sizeof(euunk), "UNK%02x", unit); + } + + if (hub) { + ec = nouveau_enum_find(nve0_fifo_fault_hubclient, client); + } else { + ec = nouveau_enum_find(nve0_fifo_fault_gpcclient, client); + snprintf(gpcid, sizeof(gpcid), "%d", gpc); + } + + if (!ec) + snprintf(ecunk, sizeof(ecunk), "UNK%02x", client); + + nv_error(priv, "%s fault at 0x%010llx [%s] from %s/%s%s%s%s on " + "channel 0x%010llx [%s]\n", write ? "write" : "read", + (u64)vahi << 32 | valo, er ? er->name : erunk, + eu ? eu->name : euunk, hub ? "" : "GPC", gpcid, hub ? "" : "/", + ec ? ec->name : ecunk, (u64)inst << 12, + nouveau_client_name(engctx)); + + object = engctx; + while (object) { + switch (nv_mclass(object)) { + case KEPLER_CHANNEL_GPFIFO_A: + nve0_fifo_recover(priv, engine, (void *)object); + break; + } + object = object->parent; + } + + nouveau_engctx_put(engctx); +} + +static const struct nouveau_bitfield nve0_fifo_pbdma_intr_0[] = { + { 0x00000001, "MEMREQ" }, + { 0x00000002, "MEMACK_TIMEOUT" }, + { 0x00000004, "MEMACK_EXTRA" }, + { 0x00000008, "MEMDAT_TIMEOUT" }, + { 0x00000010, "MEMDAT_EXTRA" }, + { 0x00000020, "MEMFLUSH" }, + { 0x00000040, "MEMOP" }, + { 0x00000080, "LBCONNECT" }, + { 0x00000100, "LBREQ" }, + { 0x00000200, "LBACK_TIMEOUT" }, + { 0x00000400, "LBACK_EXTRA" }, + { 0x00000800, "LBDAT_TIMEOUT" }, + { 0x00001000, "LBDAT_EXTRA" }, + { 0x00002000, "GPFIFO" }, + { 0x00004000, "GPPTR" }, + { 0x00008000, "GPENTRY" }, + { 0x00010000, "GPCRC" }, + { 0x00020000, "PBPTR" }, + { 0x00040000, "PBENTRY" }, + { 0x00080000, "PBCRC" }, + { 0x00100000, "XBARCONNECT" }, + { 0x00200000, "METHOD" }, + { 0x00400000, "METHODCRC" }, + { 0x00800000, "DEVICE" }, + { 0x02000000, "SEMAPHORE" }, + { 0x04000000, "ACQUIRE" }, + { 0x08000000, "PRI" }, + { 0x20000000, "NO_CTXSW_SEG" }, + { 0x40000000, "PBSEG" }, + { 0x80000000, "SIGNATURE" }, + {} +}; + +static void +nve0_fifo_intr_pbdma_0(struct nve0_fifo_priv *priv, int unit) +{ + u32 mask = nv_rd32(priv, 0x04010c + (unit * 0x2000)); + u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000)) & mask; + u32 addr = nv_rd32(priv, 0x0400c0 + (unit * 0x2000)); + u32 data = nv_rd32(priv, 0x0400c4 + (unit * 0x2000)); + u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0xfff; + u32 subc = (addr & 0x00070000) >> 16; + u32 mthd = (addr & 0x00003ffc); + u32 show = stat; + + if (stat & 0x00800000) { + if (!nve0_fifo_swmthd(priv, chid, mthd, data)) + show &= ~0x00800000; + nv_wr32(priv, 0x0400c0 + (unit * 0x2000), 0x80600008); + } + + if (show) { + nv_error(priv, "PBDMA%d:", unit); + nouveau_bitfield_print(nve0_fifo_pbdma_intr_0, show); + pr_cont("\n"); + nv_error(priv, + "PBDMA%d: ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n", + unit, chid, + nouveau_client_name_for_fifo_chid(&priv->base, chid), + subc, mthd, data); + } + + nv_wr32(priv, 0x040108 + (unit * 0x2000), stat); +} + +static const struct nouveau_bitfield nve0_fifo_pbdma_intr_1[] = { + { 0x00000001, "HCE_RE_ILLEGAL_OP" }, + { 0x00000002, "HCE_RE_ALIGNB" }, + { 0x00000004, "HCE_PRIV" }, + { 0x00000008, "HCE_ILLEGAL_MTHD" }, + { 0x00000010, "HCE_ILLEGAL_CLASS" }, + {} +}; + +static void +nve0_fifo_intr_pbdma_1(struct nve0_fifo_priv *priv, int unit) +{ + u32 mask = nv_rd32(priv, 0x04014c + (unit * 0x2000)); + u32 stat = nv_rd32(priv, 0x040148 + (unit * 0x2000)) & mask; + u32 chid = nv_rd32(priv, 0x040120 + (unit * 0x2000)) & 0xfff; + + if (stat) { + nv_error(priv, "PBDMA%d:", unit); + nouveau_bitfield_print(nve0_fifo_pbdma_intr_1, stat); + pr_cont("\n"); + nv_error(priv, "PBDMA%d: ch %d %08x %08x\n", unit, chid, + nv_rd32(priv, 0x040150 + (unit * 0x2000)), + nv_rd32(priv, 0x040154 + (unit * 0x2000))); + } + + nv_wr32(priv, 0x040148 + (unit * 0x2000), stat); +} + +static void +nve0_fifo_intr_runlist(struct nve0_fifo_priv *priv) +{ + u32 mask = nv_rd32(priv, 0x002a00); + while (mask) { + u32 engn = __ffs(mask); + wake_up(&priv->engine[engn].wait); + nv_wr32(priv, 0x002a00, 1 << engn); + mask &= ~(1 << engn); + } +} + +static void +nve0_fifo_intr_engine(struct nve0_fifo_priv *priv) +{ + nouveau_fifo_uevent(&priv->base); +} + +static void +nve0_fifo_intr(struct nouveau_subdev *subdev) +{ + struct nve0_fifo_priv *priv = (void *)subdev; + u32 mask = nv_rd32(priv, 0x002140); + u32 stat = nv_rd32(priv, 0x002100) & mask; + + if (stat & 0x00000001) { + nve0_fifo_intr_bind(priv); + nv_wr32(priv, 0x002100, 0x00000001); + stat &= ~0x00000001; + } + + if (stat & 0x00000010) { + nv_error(priv, "PIO_ERROR\n"); + nv_wr32(priv, 0x002100, 0x00000010); + stat &= ~0x00000010; + } + + if (stat & 0x00000100) { + nve0_fifo_intr_sched(priv); + nv_wr32(priv, 0x002100, 0x00000100); + stat &= ~0x00000100; + } + + if (stat & 0x00010000) { + nve0_fifo_intr_chsw(priv); + nv_wr32(priv, 0x002100, 0x00010000); + stat &= ~0x00010000; + } + + if (stat & 0x00800000) { + nv_error(priv, "FB_FLUSH_TIMEOUT\n"); + nv_wr32(priv, 0x002100, 0x00800000); + stat &= ~0x00800000; + } + + if (stat & 0x01000000) { + nv_error(priv, "LB_ERROR\n"); + nv_wr32(priv, 0x002100, 0x01000000); + stat &= ~0x01000000; + } + + if (stat & 0x08000000) { + nve0_fifo_intr_dropped_fault(priv); + nv_wr32(priv, 0x002100, 0x08000000); + stat &= ~0x08000000; + } + + if (stat & 0x10000000) { + u32 mask = nv_rd32(priv, 0x00259c); + while (mask) { + u32 unit = __ffs(mask); + nve0_fifo_intr_fault(priv, unit); + nv_wr32(priv, 0x00259c, (1 << unit)); + mask &= ~(1 << unit); + } + stat &= ~0x10000000; + } + + if (stat & 0x20000000) { + u32 mask = nv_rd32(priv, 0x0025a0); + while (mask) { + u32 unit = __ffs(mask); + nve0_fifo_intr_pbdma_0(priv, unit); + nve0_fifo_intr_pbdma_1(priv, unit); + nv_wr32(priv, 0x0025a0, (1 << unit)); + mask &= ~(1 << unit); + } + stat &= ~0x20000000; + } + + if (stat & 0x40000000) { + nve0_fifo_intr_runlist(priv); + stat &= ~0x40000000; + } + + if (stat & 0x80000000) { + nv_wr32(priv, 0x002100, 0x80000000); + nve0_fifo_intr_engine(priv); + stat &= ~0x80000000; + } + + if (stat) { + nv_error(priv, "INTR 0x%08x\n", stat); + nv_mask(priv, 0x002140, stat, 0x00000000); + nv_wr32(priv, 0x002100, stat); + } +} + +static void +nve0_fifo_uevent_init(struct nvkm_event *event, int type, int index) +{ + struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent); + nv_mask(fifo, 0x002140, 0x80000000, 0x80000000); +} + +static void +nve0_fifo_uevent_fini(struct nvkm_event *event, int type, int index) +{ + struct nouveau_fifo *fifo = container_of(event, typeof(*fifo), uevent); + nv_mask(fifo, 0x002140, 0x80000000, 0x00000000); +} + +static const struct nvkm_event_func +nve0_fifo_uevent_func = { + .ctor = nouveau_fifo_uevent_ctor, + .init = nve0_fifo_uevent_init, + .fini = nve0_fifo_uevent_fini, +}; + +int +nve0_fifo_fini(struct nouveau_object *object, bool suspend) +{ + struct nve0_fifo_priv *priv = (void *)object; + int ret; + + ret = nouveau_fifo_fini(&priv->base, suspend); + if (ret) + return ret; + + /* allow mmu fault interrupts, even when we're not using fifo */ + nv_mask(priv, 0x002140, 0x10000000, 0x10000000); + return 0; +} + +int +nve0_fifo_init(struct nouveau_object *object) +{ + struct nve0_fifo_priv *priv = (void *)object; + int ret, i; + + ret = nouveau_fifo_init(&priv->base); + if (ret) + return ret; + + /* enable all available PBDMA units */ + nv_wr32(priv, 0x000204, 0xffffffff); + priv->spoon_nr = hweight32(nv_rd32(priv, 0x000204)); + nv_debug(priv, "%d PBDMA unit(s)\n", priv->spoon_nr); + + /* PBDMA[n] */ + for (i = 0; i < priv->spoon_nr; i++) { + nv_mask(priv, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000); + nv_wr32(priv, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */ + nv_wr32(priv, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */ + } + + /* PBDMA[n].HCE */ + for (i = 0; i < priv->spoon_nr; i++) { + nv_wr32(priv, 0x040148 + (i * 0x2000), 0xffffffff); /* INTR */ + nv_wr32(priv, 0x04014c + (i * 0x2000), 0xffffffff); /* INTREN */ + } + + nv_wr32(priv, 0x002254, 0x10000000 | priv->user.bar.offset >> 12); + + nv_wr32(priv, 0x002100, 0xffffffff); + nv_wr32(priv, 0x002140, 0x7fffffff); + return 0; +} + +void +nve0_fifo_dtor(struct nouveau_object *object) +{ + struct nve0_fifo_priv *priv = (void *)object; + int i; + + nouveau_gpuobj_unmap(&priv->user.bar); + nouveau_gpuobj_ref(NULL, &priv->user.mem); + + for (i = 0; i < FIFO_ENGINE_NR; i++) { + nouveau_gpuobj_ref(NULL, &priv->engine[i].runlist[1]); + nouveau_gpuobj_ref(NULL, &priv->engine[i].runlist[0]); + } + + nouveau_fifo_destroy(&priv->base); +} + +int +nve0_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine, + struct nouveau_oclass *oclass, void *data, u32 size, + struct nouveau_object **pobject) +{ + struct nve0_fifo_impl *impl = (void *)oclass; + struct nve0_fifo_priv *priv; + int ret, i; + + ret = nouveau_fifo_create(parent, engine, oclass, 0, + impl->channels - 1, &priv); + *pobject = nv_object(priv); + if (ret) + return ret; + + INIT_WORK(&priv->fault, nve0_fifo_recover_work); + + for (i = 0; i < FIFO_ENGINE_NR; i++) { + ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x8000, 0x1000, + 0, &priv->engine[i].runlist[0]); + if (ret) + return ret; + + ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x8000, 0x1000, + 0, &priv->engine[i].runlist[1]); + if (ret) + return ret; + + init_waitqueue_head(&priv->engine[i].wait); + } + + ret = nouveau_gpuobj_new(nv_object(priv), NULL, impl->channels * 0x200, + 0x1000, NVOBJ_FLAG_ZERO_ALLOC, &priv->user.mem); + if (ret) + return ret; + + ret = nouveau_gpuobj_map(priv->user.mem, NV_MEM_ACCESS_RW, + &priv->user.bar); + if (ret) + return ret; + + ret = nvkm_event_init(&nve0_fifo_uevent_func, 1, 1, &priv->base.uevent); + if (ret) + return ret; + + nv_subdev(priv)->unit = 0x00000100; + nv_subdev(priv)->intr = nve0_fifo_intr; + nv_engine(priv)->cclass = &nve0_fifo_cclass; + nv_engine(priv)->sclass = nve0_fifo_sclass; + return 0; +} + +struct nouveau_oclass * +nve0_fifo_oclass = &(struct nve0_fifo_impl) { + .base.handle = NV_ENGINE(FIFO, 0xe0), + .base.ofuncs = &(struct nouveau_ofuncs) { + .ctor = nve0_fifo_ctor, + .dtor = nve0_fifo_dtor, + .init = nve0_fifo_init, + .fini = nve0_fifo_fini, + }, + .channels = 4096, +}.base; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nve0.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nve0.h new file mode 100644 index 000000000000..e96b32bb1bbc --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nve0.h @@ -0,0 +1,18 @@ +#ifndef __NVKM_FIFO_NVE0_H__ +#define __NVKM_FIFO_NVE0_H__ + +#include <engine/fifo.h> + +int nve0_fifo_ctor(struct nouveau_object *, struct nouveau_object *, + struct nouveau_oclass *, void *, u32, + struct nouveau_object **); +void nve0_fifo_dtor(struct nouveau_object *); +int nve0_fifo_init(struct nouveau_object *); +int nve0_fifo_fini(struct nouveau_object *, bool); + +struct nve0_fifo_impl { + struct nouveau_oclass base; + u32 channels; +}; + +#endif |