From 2fda101de0747dd7a79436cba44f17489e5a1799 Mon Sep 17 00:00:00 2001 From: Lei He Date: Sat, 8 Oct 2022 16:50:27 +0800 Subject: virtio-crypto: Support asynchronous mode virtio-crypto: Modify the current interface of virtio-crypto device to support asynchronous mode. Signed-off-by: lei he Message-Id: <20221008085030.70212-2-helei.sig11@bytedance.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- include/sysemu/cryptodev.h | 60 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 18 deletions(-) (limited to 'include') diff --git a/include/sysemu/cryptodev.h b/include/sysemu/cryptodev.h index 37c3a360fd..32e9f4cf8a 100644 --- a/include/sysemu/cryptodev.h +++ b/include/sysemu/cryptodev.h @@ -113,6 +113,7 @@ typedef struct CryptoDevBackendSessionInfo { CryptoDevBackendSymSessionInfo sym_sess_info; CryptoDevBackendAsymSessionInfo asym_sess_info; } u; + uint64_t session_id; } CryptoDevBackendSessionInfo; /** @@ -188,21 +189,30 @@ typedef struct CryptoDevBackendOpInfo { } u; } CryptoDevBackendOpInfo; +typedef void (*CryptoDevCompletionFunc) (void *opaque, int ret); struct CryptoDevBackendClass { ObjectClass parent_class; void (*init)(CryptoDevBackend *backend, Error **errp); void (*cleanup)(CryptoDevBackend *backend, Error **errp); - int64_t (*create_session)(CryptoDevBackend *backend, - CryptoDevBackendSessionInfo *sess_info, - uint32_t queue_index, Error **errp); + int (*create_session)(CryptoDevBackend *backend, + CryptoDevBackendSessionInfo *sess_info, + uint32_t queue_index, + CryptoDevCompletionFunc cb, + void *opaque); + int (*close_session)(CryptoDevBackend *backend, - uint64_t session_id, - uint32_t queue_index, Error **errp); + uint64_t session_id, + uint32_t queue_index, + CryptoDevCompletionFunc cb, + void *opaque); + int (*do_op)(CryptoDevBackend *backend, - CryptoDevBackendOpInfo *op_info, - uint32_t queue_index, Error **errp); + CryptoDevBackendOpInfo *op_info, + uint32_t queue_index, + CryptoDevCompletionFunc cb, + void *opaque); }; typedef enum CryptoDevBackendOptionsType { @@ -303,15 +313,20 @@ void cryptodev_backend_cleanup( * @sess_info: parameters needed by session creating * @queue_index: queue index of cryptodev backend client * @errp: pointer to a NULL-initialized error object + * @cb: callback when session create is compeleted + * @opaque: parameter passed to callback * - * Create a session for symmetric/symmetric algorithms + * Create a session for symmetric/asymmetric algorithms * - * Returns: session id on success, or -1 on error + * Returns: 0 for success and cb will be called when creation is completed, + * negative value for error, and cb will not be called. */ -int64_t cryptodev_backend_create_session( +int cryptodev_backend_create_session( CryptoDevBackend *backend, CryptoDevBackendSessionInfo *sess_info, - uint32_t queue_index, Error **errp); + uint32_t queue_index, + CryptoDevCompletionFunc cb, + void *opaque); /** * cryptodev_backend_close_session: @@ -319,34 +334,43 @@ int64_t cryptodev_backend_create_session( * @session_id: the session id * @queue_index: queue index of cryptodev backend client * @errp: pointer to a NULL-initialized error object + * @cb: callback when session create is compeleted + * @opaque: parameter passed to callback * * Close a session for which was previously * created by cryptodev_backend_create_session() * - * Returns: 0 on success, or Negative on error + * Returns: 0 for success and cb will be called when creation is completed, + * negative value for error, and cb will not be called. */ int cryptodev_backend_close_session( CryptoDevBackend *backend, uint64_t session_id, - uint32_t queue_index, Error **errp); + uint32_t queue_index, + CryptoDevCompletionFunc cb, + void *opaque); /** * cryptodev_backend_crypto_operation: * @backend: the cryptodev backend object - * @opaque: pointer to a VirtIOCryptoReq object + * @opaque1: pointer to a VirtIOCryptoReq object * @queue_index: queue index of cryptodev backend client * @errp: pointer to a NULL-initialized error object + * @cb: callbacks when operation is completed + * @opaque2: parameter passed to cb * * Do crypto operation, such as encryption and * decryption * - * Returns: VIRTIO_CRYPTO_OK on success, - * or -VIRTIO_CRYPTO_* on error + * Returns: 0 for success and cb will be called when creation is completed, + * negative value for error, and cb will not be called. */ int cryptodev_backend_crypto_operation( CryptoDevBackend *backend, - void *opaque, - uint32_t queue_index, Error **errp); + void *opaque1, + uint32_t queue_index, + CryptoDevCompletionFunc cb, + void *opaque2); /** * cryptodev_backend_set_used: -- cgit v1.2.3-55-g7522 From 58660863ba5ca4f74fa70671da2899b264dc5f34 Mon Sep 17 00:00:00 2001 From: Lei He Date: Sat, 8 Oct 2022 16:50:29 +0800 Subject: crypto: Support export akcipher to pkcs8 crypto: support export RSA private keys with PKCS#8 standard. So that users can upload this private key to linux kernel. Signed-off-by: lei he Message-Id: <20221008085030.70212-4-helei.sig11@bytedance.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin Reviewed-by: Daniel P. Berrangé --- crypto/akcipher.c | 18 ++++++++++++++++++ crypto/rsakey.c | 42 ++++++++++++++++++++++++++++++++++++++++++ crypto/rsakey.h | 11 ++++++++++- include/crypto/akcipher.h | 21 +++++++++++++++++++++ 4 files changed, 91 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/crypto/akcipher.c b/crypto/akcipher.c index ad88379c1e..e4bbc6e5f1 100644 --- a/crypto/akcipher.c +++ b/crypto/akcipher.c @@ -22,6 +22,8 @@ #include "qemu/osdep.h" #include "crypto/akcipher.h" #include "akcipherpriv.h" +#include "der.h" +#include "rsakey.h" #if defined(CONFIG_GCRYPT) #include "akcipher-gcrypt.c.inc" @@ -106,3 +108,19 @@ void qcrypto_akcipher_free(QCryptoAkCipher *akcipher) drv->free(akcipher); } + +int qcrypto_akcipher_export_p8info(const QCryptoAkCipherOptions *opts, + uint8_t *key, size_t keylen, + uint8_t **dst, size_t *dst_len, + Error **errp) +{ + switch (opts->alg) { + case QCRYPTO_AKCIPHER_ALG_RSA: + qcrypto_akcipher_rsakey_export_p8info(key, keylen, dst, dst_len); + return 0; + + default: + error_setg(errp, "Unsupported algorithm: %u", opts->alg); + return -1; + } +} diff --git a/crypto/rsakey.c b/crypto/rsakey.c index cc40e072f0..7d6f273aef 100644 --- a/crypto/rsakey.c +++ b/crypto/rsakey.c @@ -19,6 +19,8 @@ * */ +#include "qemu/osdep.h" +#include "der.h" #include "rsakey.h" void qcrypto_akcipher_rsakey_free(QCryptoAkCipherRSAKey *rsa_key) @@ -37,6 +39,46 @@ void qcrypto_akcipher_rsakey_free(QCryptoAkCipherRSAKey *rsa_key) g_free(rsa_key); } +/** + * PKCS#8 private key info for RSA + * + * PrivateKeyInfo ::= SEQUENCE { + * version INTEGER, + * privateKeyAlgorithm PrivateKeyAlgorithmIdentifier, + * privateKey OCTET STRING, + * attributes [0] IMPLICIT Attributes OPTIONAL + * } + */ +void qcrypto_akcipher_rsakey_export_p8info(const uint8_t *key, + size_t keylen, + uint8_t **dst, + size_t *dlen) +{ + QCryptoEncodeContext *ctx = qcrypto_der_encode_ctx_new(); + uint8_t version = 0; + + qcrypto_der_encode_seq_begin(ctx); + + /* version */ + qcrypto_der_encode_int(ctx, &version, sizeof(version)); + + /* algorithm identifier */ + qcrypto_der_encode_seq_begin(ctx); + qcrypto_der_encode_oid(ctx, (uint8_t *)QCRYPTO_OID_rsaEncryption, + sizeof(QCRYPTO_OID_rsaEncryption) - 1); + qcrypto_der_encode_null(ctx); + qcrypto_der_encode_seq_end(ctx); + + /* RSA private key */ + qcrypto_der_encode_octet_str(ctx, key, keylen); + + qcrypto_der_encode_seq_end(ctx); + + *dlen = qcrypto_der_encode_ctx_buffer_len(ctx); + *dst = g_malloc(*dlen); + qcrypto_der_encode_ctx_flush_and_free(ctx, *dst); +} + #if defined(CONFIG_NETTLE) && defined(CONFIG_HOGWEED) #include "rsakey-nettle.c.inc" #else diff --git a/crypto/rsakey.h b/crypto/rsakey.h index 974b76f659..00b3eccec7 100644 --- a/crypto/rsakey.h +++ b/crypto/rsakey.h @@ -22,7 +22,6 @@ #ifndef QCRYPTO_RSAKEY_H #define QCRYPTO_RSAKEY_H -#include "qemu/osdep.h" #include "qemu/host-utils.h" #include "crypto/akcipher.h" @@ -84,6 +83,16 @@ QCryptoAkCipherRSAKey *qcrypto_akcipher_rsakey_parse( QCryptoAkCipherKeyType type, const uint8_t *key, size_t keylen, Error **errp); +/** + * qcrypto_akcipher_rsakey_export_as_p8info: + * + * Export RSA private key to PKCS#8 private key info. + */ +void qcrypto_akcipher_rsakey_export_p8info(const uint8_t *key, + size_t keylen, + uint8_t **dst, + size_t *dlen); + void qcrypto_akcipher_rsakey_free(QCryptoAkCipherRSAKey *key); G_DEFINE_AUTOPTR_CLEANUP_FUNC(QCryptoAkCipherRSAKey, diff --git a/include/crypto/akcipher.h b/include/crypto/akcipher.h index 51f5fa2774..214e58ca47 100644 --- a/include/crypto/akcipher.h +++ b/include/crypto/akcipher.h @@ -153,6 +153,27 @@ int qcrypto_akcipher_max_dgst_len(QCryptoAkCipher *akcipher); */ void qcrypto_akcipher_free(QCryptoAkCipher *akcipher); +/** + * qcrypto_akcipher_export_p8info: + * @opts: the options of the akcipher to be exported. + * @key: the original key of the akcipher to be exported. + * @keylen: length of the 'key' + * @dst: output parameter, if export succeed, *dst is set to the + * PKCS#8 encoded private key, caller MUST free this key with + * g_free after use. + * @dst_len: output parameter, indicates the length of PKCS#8 encoded + * key. + * + * Export the akcipher into DER encoded pkcs#8 private key info, expects + * |key| stores a valid asymmetric PRIVATE key. + * + * Returns: 0 for succeed, otherwise -1 is returned. + */ +int qcrypto_akcipher_export_p8info(const QCryptoAkCipherOptions *opts, + uint8_t *key, size_t keylen, + uint8_t **dst, size_t *dst_len, + Error **errp); + G_DEFINE_AUTOPTR_CLEANUP_FUNC(QCryptoAkCipher, qcrypto_akcipher_free) #endif /* QCRYPTO_AKCIPHER_H */ -- cgit v1.2.3-55-g7522 From 39fff6f3e8b3dd4f1708f8bdde0a51adbe63188a Mon Sep 17 00:00:00 2001 From: Lei He Date: Sat, 8 Oct 2022 16:50:30 +0800 Subject: cryptodev: Add a lkcf-backend for cryptodev cryptodev: Added a new type of backend named lkcf-backend for cryptodev. This backend upload asymmetric keys to linux kernel, and let kernel do the accelerations if possible. The lkcf stands for Linux Kernel Cryptography Framework. Signed-off-by: lei he Message-Id: <20221008085030.70212-5-helei.sig11@bytedance.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- backends/cryptodev-lkcf.c | 645 +++++++++++++++++++++++++++++++++++++++++++++ backends/meson.build | 3 + include/sysemu/cryptodev.h | 1 + qapi/qom.json | 2 + 4 files changed, 651 insertions(+) create mode 100644 backends/cryptodev-lkcf.c (limited to 'include') diff --git a/backends/cryptodev-lkcf.c b/backends/cryptodev-lkcf.c new file mode 100644 index 0000000000..133bd706a4 --- /dev/null +++ b/backends/cryptodev-lkcf.c @@ -0,0 +1,645 @@ +/* + * QEMU Cryptodev backend for QEMU cipher APIs + * + * Copyright (c) 2022 Bytedance.Inc + * + * Authors: + * lei he + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + * + */ + +#include "qemu/osdep.h" +#include "crypto/cipher.h" +#include "crypto/akcipher.h" +#include "qapi/error.h" +#include "qemu/main-loop.h" +#include "qemu/thread.h" +#include "qemu/error-report.h" +#include "qemu/queue.h" +#include "qom/object.h" +#include "sysemu/cryptodev.h" +#include "standard-headers/linux/virtio_crypto.h" + +#include +#include + +/** + * @TYPE_CRYPTODEV_BACKEND_LKCF: + * name of backend that uses linux kernel crypto framework + */ +#define TYPE_CRYPTODEV_BACKEND_LKCF "cryptodev-backend-lkcf" + +OBJECT_DECLARE_SIMPLE_TYPE(CryptoDevBackendLKCF, CRYPTODEV_BACKEND_LKCF) + +#define INVALID_KEY_ID -1 +#define MAX_SESSIONS 256 +#define NR_WORKER_THREAD 64 + +#define KCTL_KEY_TYPE_PKEY "asymmetric" +/** + * Here the key is uploaded to the thread-keyring of worker thread, at least + * util linux-6.0: + * 1. process keyring seems to behave unexpectedly if main-thread does not + * create the keyring before creating any other thread. + * 2. at present, the guest kernel never perform multiple operations on a + * session. + * 3. it can reduce the load of the main-loop because the key passed by the + * guest kernel has been already checked. + */ +#define KCTL_KEY_RING KEY_SPEC_THREAD_KEYRING + +typedef struct CryptoDevBackendLKCFSession { + uint8_t *key; + size_t keylen; + QCryptoAkCipherKeyType keytype; + QCryptoAkCipherOptions akcipher_opts; +} CryptoDevBackendLKCFSession; + +typedef struct CryptoDevBackendLKCF CryptoDevBackendLKCF; +typedef struct CryptoDevLKCFTask CryptoDevLKCFTask; +struct CryptoDevLKCFTask { + CryptoDevBackendLKCFSession *sess; + CryptoDevBackendOpInfo *op_info; + CryptoDevCompletionFunc cb; + void *opaque; + int status; + CryptoDevBackendLKCF *lkcf; + QSIMPLEQ_ENTRY(CryptoDevLKCFTask) queue; +}; + +typedef struct CryptoDevBackendLKCF { + CryptoDevBackend parent_obj; + CryptoDevBackendLKCFSession *sess[MAX_SESSIONS]; + QSIMPLEQ_HEAD(, CryptoDevLKCFTask) requests; + QSIMPLEQ_HEAD(, CryptoDevLKCFTask) responses; + QemuMutex mutex; + QemuCond cond; + QemuMutex rsp_mutex; + + /** + * There is no async interface for asymmetric keys like AF_ALG sockets, + * we don't seem to have better way than create a lots of thread. + */ + QemuThread worker_threads[NR_WORKER_THREAD]; + bool running; + int eventfd; +} CryptoDevBackendLKCF; + +static void *cryptodev_lkcf_worker(void *arg); +static int cryptodev_lkcf_close_session(CryptoDevBackend *backend, + uint64_t session_id, + uint32_t queue_index, + CryptoDevCompletionFunc cb, + void *opaque); + +static void cryptodev_lkcf_handle_response(void *opaque) +{ + CryptoDevBackendLKCF *lkcf = (CryptoDevBackendLKCF *)opaque; + QSIMPLEQ_HEAD(, CryptoDevLKCFTask) responses; + CryptoDevLKCFTask *task, *next; + eventfd_t nevent; + + QSIMPLEQ_INIT(&responses); + eventfd_read(lkcf->eventfd, &nevent); + + qemu_mutex_lock(&lkcf->rsp_mutex); + QSIMPLEQ_PREPEND(&responses, &lkcf->responses); + qemu_mutex_unlock(&lkcf->rsp_mutex); + + QSIMPLEQ_FOREACH_SAFE(task, &responses, queue, next) { + if (task->cb) { + task->cb(task->opaque, task->status); + } + g_free(task); + } +} + +static int cryptodev_lkcf_set_op_desc(QCryptoAkCipherOptions *opts, + char *key_desc, + size_t desc_len, + Error **errp) +{ + QCryptoAkCipherOptionsRSA *rsa_opt; + if (opts->alg != QCRYPTO_AKCIPHER_ALG_RSA) { + error_setg(errp, "Unsupported alg: %u", opts->alg); + return -1; + } + + rsa_opt = &opts->u.rsa; + if (rsa_opt->padding_alg == QCRYPTO_RSA_PADDING_ALG_PKCS1) { + snprintf(key_desc, desc_len, "enc=%s hash=%s", + QCryptoRSAPaddingAlgorithm_str(rsa_opt->padding_alg), + QCryptoHashAlgorithm_str(rsa_opt->hash_alg)); + + } else { + snprintf(key_desc, desc_len, "enc=%s", + QCryptoRSAPaddingAlgorithm_str(rsa_opt->padding_alg)); + } + return 0; +} + +static int cryptodev_lkcf_set_rsa_opt(int virtio_padding_alg, + int virtio_hash_alg, + QCryptoAkCipherOptionsRSA *opt, + Error **errp) +{ + if (virtio_padding_alg == VIRTIO_CRYPTO_RSA_PKCS1_PADDING) { + opt->padding_alg = QCRYPTO_RSA_PADDING_ALG_PKCS1; + + switch (virtio_hash_alg) { + case VIRTIO_CRYPTO_RSA_MD5: + opt->hash_alg = QCRYPTO_HASH_ALG_MD5; + break; + + case VIRTIO_CRYPTO_RSA_SHA1: + opt->hash_alg = QCRYPTO_HASH_ALG_SHA1; + break; + + case VIRTIO_CRYPTO_RSA_SHA256: + opt->hash_alg = QCRYPTO_HASH_ALG_SHA256; + break; + + case VIRTIO_CRYPTO_RSA_SHA512: + opt->hash_alg = QCRYPTO_HASH_ALG_SHA512; + break; + + default: + error_setg(errp, "Unsupported rsa hash algo: %d", virtio_hash_alg); + return -1; + } + return 0; + } + + if (virtio_padding_alg == VIRTIO_CRYPTO_RSA_RAW_PADDING) { + opt->padding_alg = QCRYPTO_RSA_PADDING_ALG_RAW; + return 0; + } + + error_setg(errp, "Unsupported rsa padding algo: %u", virtio_padding_alg); + return -1; +} + +static int cryptodev_lkcf_get_unused_session_index(CryptoDevBackendLKCF *lkcf) +{ + size_t i; + + for (i = 0; i < MAX_SESSIONS; i++) { + if (lkcf->sess[i] == NULL) { + return i; + } + } + return -1; +} + +static void cryptodev_lkcf_init(CryptoDevBackend *backend, Error **errp) +{ + /* Only support one queue */ + int queues = backend->conf.peers.queues, i; + CryptoDevBackendClient *cc; + CryptoDevBackendLKCF *lkcf = + CRYPTODEV_BACKEND_LKCF(backend); + + if (queues != 1) { + error_setg(errp, + "Only support one queue in cryptodev-builtin backend"); + return; + } + lkcf->eventfd = eventfd(0, 0); + if (lkcf->eventfd < 0) { + error_setg(errp, "Failed to create eventfd: %d", errno); + return; + } + + cc = cryptodev_backend_new_client("cryptodev-lkcf", NULL); + cc->info_str = g_strdup_printf("cryptodev-lkcf0"); + cc->queue_index = 0; + cc->type = CRYPTODEV_BACKEND_TYPE_LKCF; + backend->conf.peers.ccs[0] = cc; + + backend->conf.crypto_services = + 1u << VIRTIO_CRYPTO_SERVICE_AKCIPHER; + backend->conf.akcipher_algo = 1u << VIRTIO_CRYPTO_AKCIPHER_RSA; + lkcf->running = true; + + QSIMPLEQ_INIT(&lkcf->requests); + QSIMPLEQ_INIT(&lkcf->responses); + qemu_mutex_init(&lkcf->mutex); + qemu_mutex_init(&lkcf->rsp_mutex); + qemu_cond_init(&lkcf->cond); + for (i = 0; i < NR_WORKER_THREAD; i++) { + qemu_thread_create(&lkcf->worker_threads[i], "lkcf-worker", + cryptodev_lkcf_worker, lkcf, 0); + } + qemu_set_fd_handler( + lkcf->eventfd, cryptodev_lkcf_handle_response, NULL, lkcf); + cryptodev_backend_set_ready(backend, true); +} + +static void cryptodev_lkcf_cleanup(CryptoDevBackend *backend, Error **errp) +{ + CryptoDevBackendLKCF *lkcf = CRYPTODEV_BACKEND_LKCF(backend); + size_t i; + int queues = backend->conf.peers.queues; + CryptoDevBackendClient *cc; + CryptoDevLKCFTask *task, *next; + + qemu_mutex_lock(&lkcf->mutex); + lkcf->running = false; + qemu_mutex_unlock(&lkcf->mutex); + qemu_cond_broadcast(&lkcf->cond); + + close(lkcf->eventfd); + for (i = 0; i < NR_WORKER_THREAD; i++) { + qemu_thread_join(&lkcf->worker_threads[i]); + } + + QSIMPLEQ_FOREACH_SAFE(task, &lkcf->requests, queue, next) { + if (task->cb) { + task->cb(task->opaque, task->status); + } + g_free(task); + } + + QSIMPLEQ_FOREACH_SAFE(task, &lkcf->responses, queue, next) { + if (task->cb) { + task->cb(task->opaque, task->status); + } + g_free(task); + } + + qemu_mutex_destroy(&lkcf->mutex); + qemu_cond_destroy(&lkcf->cond); + qemu_mutex_destroy(&lkcf->rsp_mutex); + + for (i = 0; i < MAX_SESSIONS; i++) { + if (lkcf->sess[i] != NULL) { + cryptodev_lkcf_close_session(backend, i, 0, NULL, NULL); + } + } + + for (i = 0; i < queues; i++) { + cc = backend->conf.peers.ccs[i]; + if (cc) { + cryptodev_backend_free_client(cc); + backend->conf.peers.ccs[i] = NULL; + } + } + + cryptodev_backend_set_ready(backend, false); +} + +static void cryptodev_lkcf_execute_task(CryptoDevLKCFTask *task) +{ + CryptoDevBackendLKCFSession *session = task->sess; + CryptoDevBackendAsymOpInfo *asym_op_info; + bool kick = false; + int ret, status, op_code = task->op_info->op_code; + size_t p8info_len; + g_autofree uint8_t *p8info = NULL; + Error *local_error = NULL; + key_serial_t key_id = INVALID_KEY_ID; + char op_desc[64]; + g_autoptr(QCryptoAkCipher) akcipher = NULL; + + /** + * We only offload private key session: + * 1. currently, the Linux kernel can only accept public key wrapped + * with X.509 certificates, but unfortunately the cost of making a + * ceritificate with public key is too expensive. + * 2. generally, public key related compution is fast, just compute it with + * thread-pool. + */ + if (session->keytype == QCRYPTO_AKCIPHER_KEY_TYPE_PRIVATE) { + if (qcrypto_akcipher_export_p8info(&session->akcipher_opts, + session->key, session->keylen, + &p8info, &p8info_len, + &local_error) != 0 || + cryptodev_lkcf_set_op_desc(&session->akcipher_opts, op_desc, + sizeof(op_desc), &local_error) != 0) { + error_report_err(local_error); + } else { + key_id = add_key(KCTL_KEY_TYPE_PKEY, "lkcf-backend-priv-key", + p8info, p8info_len, KCTL_KEY_RING); + } + } + + if (key_id < 0) { + if (!qcrypto_akcipher_supports(&session->akcipher_opts)) { + status = -VIRTIO_CRYPTO_NOTSUPP; + goto out; + } + akcipher = qcrypto_akcipher_new(&session->akcipher_opts, + session->keytype, + session->key, session->keylen, + &local_error); + if (!akcipher) { + status = -VIRTIO_CRYPTO_ERR; + goto out; + } + } + + asym_op_info = task->op_info->u.asym_op_info; + switch (op_code) { + case VIRTIO_CRYPTO_AKCIPHER_ENCRYPT: + if (key_id >= 0) { + ret = keyctl_pkey_encrypt(key_id, op_desc, + asym_op_info->src, asym_op_info->src_len, + asym_op_info->dst, asym_op_info->dst_len); + } else { + ret = qcrypto_akcipher_encrypt(akcipher, + asym_op_info->src, asym_op_info->src_len, + asym_op_info->dst, asym_op_info->dst_len, &local_error); + } + break; + + case VIRTIO_CRYPTO_AKCIPHER_DECRYPT: + if (key_id >= 0) { + ret = keyctl_pkey_decrypt(key_id, op_desc, + asym_op_info->src, asym_op_info->src_len, + asym_op_info->dst, asym_op_info->dst_len); + } else { + ret = qcrypto_akcipher_decrypt(akcipher, + asym_op_info->src, asym_op_info->src_len, + asym_op_info->dst, asym_op_info->dst_len, &local_error); + } + break; + + case VIRTIO_CRYPTO_AKCIPHER_SIGN: + if (key_id >= 0) { + ret = keyctl_pkey_sign(key_id, op_desc, + asym_op_info->src, asym_op_info->src_len, + asym_op_info->dst, asym_op_info->dst_len); + } else { + ret = qcrypto_akcipher_sign(akcipher, + asym_op_info->src, asym_op_info->src_len, + asym_op_info->dst, asym_op_info->dst_len, &local_error); + } + break; + + case VIRTIO_CRYPTO_AKCIPHER_VERIFY: + if (key_id >= 0) { + ret = keyctl_pkey_verify(key_id, op_desc, + asym_op_info->src, asym_op_info->src_len, + asym_op_info->dst, asym_op_info->dst_len); + } else { + ret = qcrypto_akcipher_verify(akcipher, + asym_op_info->src, asym_op_info->src_len, + asym_op_info->dst, asym_op_info->dst_len, &local_error); + } + break; + + default: + error_setg(&local_error, "Unknown opcode: %u", op_code); + status = -VIRTIO_CRYPTO_ERR; + goto out; + } + + if (ret < 0) { + if (!local_error) { + if (errno != EKEYREJECTED) { + error_report("Failed do operation with keyctl: %d", errno); + } + } else { + error_report_err(local_error); + } + status = op_code == VIRTIO_CRYPTO_AKCIPHER_VERIFY ? + -VIRTIO_CRYPTO_KEY_REJECTED : -VIRTIO_CRYPTO_ERR; + } else { + status = VIRTIO_CRYPTO_OK; + asym_op_info->dst_len = ret; + } + +out: + if (key_id >= 0) { + keyctl_unlink(key_id, KCTL_KEY_RING); + } + task->status = status; + + qemu_mutex_lock(&task->lkcf->rsp_mutex); + if (QSIMPLEQ_EMPTY(&task->lkcf->responses)) { + kick = true; + } + QSIMPLEQ_INSERT_TAIL(&task->lkcf->responses, task, queue); + qemu_mutex_unlock(&task->lkcf->rsp_mutex); + + if (kick) { + eventfd_write(task->lkcf->eventfd, 1); + } +} + +static void *cryptodev_lkcf_worker(void *arg) +{ + CryptoDevBackendLKCF *backend = (CryptoDevBackendLKCF *)arg; + CryptoDevLKCFTask *task; + + for (;;) { + task = NULL; + qemu_mutex_lock(&backend->mutex); + while (backend->running && QSIMPLEQ_EMPTY(&backend->requests)) { + qemu_cond_wait(&backend->cond, &backend->mutex); + } + if (backend->running) { + task = QSIMPLEQ_FIRST(&backend->requests); + QSIMPLEQ_REMOVE_HEAD(&backend->requests, queue); + } + qemu_mutex_unlock(&backend->mutex); + + /* stopped */ + if (!task) { + break; + } + cryptodev_lkcf_execute_task(task); + } + + return NULL; +} + +static int cryptodev_lkcf_operation( + CryptoDevBackend *backend, + CryptoDevBackendOpInfo *op_info, + uint32_t queue_index, + CryptoDevCompletionFunc cb, + void *opaque) +{ + CryptoDevBackendLKCF *lkcf = + CRYPTODEV_BACKEND_LKCF(backend); + CryptoDevBackendLKCFSession *sess; + enum CryptoDevBackendAlgType algtype = op_info->algtype; + CryptoDevLKCFTask *task; + + if (op_info->session_id >= MAX_SESSIONS || + lkcf->sess[op_info->session_id] == NULL) { + error_report("Cannot find a valid session id: %" PRIu64 "", + op_info->session_id); + return -VIRTIO_CRYPTO_INVSESS; + } + + sess = lkcf->sess[op_info->session_id]; + if (algtype != CRYPTODEV_BACKEND_ALG_ASYM) { + error_report("algtype not supported: %u", algtype); + return -VIRTIO_CRYPTO_NOTSUPP; + } + + task = g_new0(CryptoDevLKCFTask, 1); + task->op_info = op_info; + task->cb = cb; + task->opaque = opaque; + task->sess = sess; + task->lkcf = lkcf; + task->status = -VIRTIO_CRYPTO_ERR; + + qemu_mutex_lock(&lkcf->mutex); + QSIMPLEQ_INSERT_TAIL(&lkcf->requests, task, queue); + qemu_mutex_unlock(&lkcf->mutex); + qemu_cond_signal(&lkcf->cond); + + return VIRTIO_CRYPTO_OK; +} + +static int cryptodev_lkcf_create_asym_session( + CryptoDevBackendLKCF *lkcf, + CryptoDevBackendAsymSessionInfo *sess_info, + uint64_t *session_id) +{ + Error *local_error = NULL; + int index; + g_autofree CryptoDevBackendLKCFSession *sess = + g_new0(CryptoDevBackendLKCFSession, 1); + + switch (sess_info->algo) { + case VIRTIO_CRYPTO_AKCIPHER_RSA: + sess->akcipher_opts.alg = QCRYPTO_AKCIPHER_ALG_RSA; + if (cryptodev_lkcf_set_rsa_opt( + sess_info->u.rsa.padding_algo, sess_info->u.rsa.hash_algo, + &sess->akcipher_opts.u.rsa, &local_error) != 0) { + error_report_err(local_error); + return -VIRTIO_CRYPTO_ERR; + } + break; + + default: + error_report("Unsupported asym alg %u", sess_info->algo); + return -VIRTIO_CRYPTO_NOTSUPP; + } + + switch (sess_info->keytype) { + case VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PUBLIC: + sess->keytype = QCRYPTO_AKCIPHER_KEY_TYPE_PUBLIC; + break; + + case VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PRIVATE: + sess->keytype = QCRYPTO_AKCIPHER_KEY_TYPE_PRIVATE; + break; + + default: + error_report("Unknown akcipher keytype: %u", sess_info->keytype); + return -VIRTIO_CRYPTO_ERR; + } + + index = cryptodev_lkcf_get_unused_session_index(lkcf); + if (index < 0) { + error_report("Total number of sessions created exceeds %u", + MAX_SESSIONS); + return -VIRTIO_CRYPTO_ERR; + } + + sess->keylen = sess_info->keylen; + sess->key = g_malloc(sess_info->keylen); + memcpy(sess->key, sess_info->key, sess_info->keylen); + + lkcf->sess[index] = g_steal_pointer(&sess); + *session_id = index; + + return VIRTIO_CRYPTO_OK; +} + +static int cryptodev_lkcf_create_session( + CryptoDevBackend *backend, + CryptoDevBackendSessionInfo *sess_info, + uint32_t queue_index, + CryptoDevCompletionFunc cb, + void *opaque) +{ + CryptoDevBackendAsymSessionInfo *asym_sess_info; + CryptoDevBackendLKCF *lkcf = + CRYPTODEV_BACKEND_LKCF(backend); + int ret; + + switch (sess_info->op_code) { + case VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION: + asym_sess_info = &sess_info->u.asym_sess_info; + ret = cryptodev_lkcf_create_asym_session( + lkcf, asym_sess_info, &sess_info->session_id); + break; + + default: + ret = -VIRTIO_CRYPTO_NOTSUPP; + error_report("Unsupported opcode: %" PRIu32 "", + sess_info->op_code); + break; + } + if (cb) { + cb(opaque, ret); + } + return 0; +} + +static int cryptodev_lkcf_close_session(CryptoDevBackend *backend, + uint64_t session_id, + uint32_t queue_index, + CryptoDevCompletionFunc cb, + void *opaque) +{ + CryptoDevBackendLKCF *lkcf = CRYPTODEV_BACKEND_LKCF(backend); + CryptoDevBackendLKCFSession *session; + + assert(session_id < MAX_SESSIONS && lkcf->sess[session_id]); + session = lkcf->sess[session_id]; + lkcf->sess[session_id] = NULL; + + g_free(session->key); + g_free(session); + + if (cb) { + cb(opaque, VIRTIO_CRYPTO_OK); + } + return 0; +} + +static void cryptodev_lkcf_class_init(ObjectClass *oc, void *data) +{ + CryptoDevBackendClass *bc = CRYPTODEV_BACKEND_CLASS(oc); + + bc->init = cryptodev_lkcf_init; + bc->cleanup = cryptodev_lkcf_cleanup; + bc->create_session = cryptodev_lkcf_create_session; + bc->close_session = cryptodev_lkcf_close_session; + bc->do_op = cryptodev_lkcf_operation; +} + +static const TypeInfo cryptodev_builtin_info = { + .name = TYPE_CRYPTODEV_BACKEND_LKCF, + .parent = TYPE_CRYPTODEV_BACKEND, + .class_init = cryptodev_lkcf_class_init, + .instance_size = sizeof(CryptoDevBackendLKCF), +}; + +static void cryptodev_lkcf_register_types(void) +{ + type_register_static(&cryptodev_builtin_info); +} + +type_init(cryptodev_lkcf_register_types); diff --git a/backends/meson.build b/backends/meson.build index b1884a88ec..954e658b25 100644 --- a/backends/meson.build +++ b/backends/meson.build @@ -12,6 +12,9 @@ softmmu_ss.add([files( softmmu_ss.add(when: 'CONFIG_POSIX', if_true: files('rng-random.c')) softmmu_ss.add(when: 'CONFIG_POSIX', if_true: files('hostmem-file.c')) softmmu_ss.add(when: 'CONFIG_LINUX', if_true: files('hostmem-memfd.c')) +if keyutils.found() + softmmu_ss.add(keyutils, files('cryptodev-lkcf.c')) +endif if have_vhost_user softmmu_ss.add(when: 'CONFIG_VIRTIO', if_true: files('vhost-user.c')) endif diff --git a/include/sysemu/cryptodev.h b/include/sysemu/cryptodev.h index 32e9f4cf8a..cf9b3f07fe 100644 --- a/include/sysemu/cryptodev.h +++ b/include/sysemu/cryptodev.h @@ -219,6 +219,7 @@ typedef enum CryptoDevBackendOptionsType { CRYPTODEV_BACKEND_TYPE_NONE = 0, CRYPTODEV_BACKEND_TYPE_BUILTIN = 1, CRYPTODEV_BACKEND_TYPE_VHOST_USER = 2, + CRYPTODEV_BACKEND_TYPE_LKCF = 3, CRYPTODEV_BACKEND_TYPE__MAX, } CryptoDevBackendOptionsType; diff --git a/qapi/qom.json b/qapi/qom.json index 87fcad2423..d2e0244e57 100644 --- a/qapi/qom.json +++ b/qapi/qom.json @@ -876,6 +876,7 @@ 'colo-compare', 'cryptodev-backend', 'cryptodev-backend-builtin', + 'cryptodev-backend-lkcf', { 'name': 'cryptodev-vhost-user', 'if': 'CONFIG_VHOST_CRYPTO' }, 'dbus-vmstate', @@ -944,6 +945,7 @@ 'colo-compare': 'ColoCompareProperties', 'cryptodev-backend': 'CryptodevBackendProperties', 'cryptodev-backend-builtin': 'CryptodevBackendProperties', + 'cryptodev-backend-lkcf': 'CryptodevBackendProperties', 'cryptodev-vhost-user': { 'type': 'CryptodevVhostUserProperties', 'if': 'CONFIG_VHOST_CRYPTO' }, 'dbus-vmstate': 'DBusVMStateProperties', -- cgit v1.2.3-55-g7522 From 5fb52f6cc8f621f2e51d181d81401d14e4d45102 Mon Sep 17 00:00:00 2001 From: Huai-Cheng Kuo Date: Fri, 14 Oct 2022 16:10:41 +0100 Subject: hw/pci: PCIe Data Object Exchange emulation Emulation of PCIe Data Object Exchange (DOE) PCIE Base Specification r6.0 6.3 Data Object Exchange Supports multiple DOE PCIe Extended Capabilities for a single PCIe device. For each capability, a static array of DOEProtocol should be passed to pcie_doe_init(). The protocols in that array will be registered under the DOE capability structure. For each protocol, vendor ID, type, and corresponding callback function (handle_request()) should be implemented. This callback function represents how the DOE request for corresponding protocol will be handled. pcie_doe_{read/write}_config() must be appended to corresponding PCI device's config_read/write() handler to enable DOE access. In pcie_doe_read_config(), false will be returned if pci_config_read() offset is not within DOE capability range. In pcie_doe_write_config(), the function will have no affect if the address is not within the related DOE PCIE extended capability. Signed-off-by: Huai-Cheng Kuo Signed-off-by: Chris Browy Signed-off-by: Jonathan Cameron Message-Id: <20221014151045.24781-2-Jonathan.Cameron@huawei.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- MAINTAINERS | 7 + hw/pci/meson.build | 1 + hw/pci/pcie_doe.c | 367 +++++++++++++++++++++++++++++++++++++++++++++ include/hw/pci/pci_ids.h | 3 + include/hw/pci/pcie.h | 1 + include/hw/pci/pcie_doe.h | 123 +++++++++++++++ include/hw/pci/pcie_regs.h | 4 + 7 files changed, 506 insertions(+) create mode 100644 hw/pci/pcie_doe.c create mode 100644 include/hw/pci/pcie_doe.h (limited to 'include') diff --git a/MAINTAINERS b/MAINTAINERS index 07df572adf..8b7d49b089 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1835,6 +1835,13 @@ F: qapi/pci.json F: docs/pci* F: docs/specs/*pci* +PCIE DOE +M: Huai-Cheng Kuo +M: Chris Browy +S: Supported +F: include/hw/pci/pcie_doe.h +F: hw/pci/pcie_doe.c + ACPI/SMBIOS M: Michael S. Tsirkin M: Igor Mammedov diff --git a/hw/pci/meson.build b/hw/pci/meson.build index bcc9c75919..5aff7ed1c6 100644 --- a/hw/pci/meson.build +++ b/hw/pci/meson.build @@ -13,6 +13,7 @@ pci_ss.add(files( # allow plugging PCIe devices into PCI buses, include them even if # CONFIG_PCI_EXPRESS=n. pci_ss.add(files('pcie.c', 'pcie_aer.c')) +pci_ss.add(files('pcie_doe.c')) softmmu_ss.add(when: 'CONFIG_PCI_EXPRESS', if_true: files('pcie_port.c', 'pcie_host.c')) softmmu_ss.add_all(when: 'CONFIG_PCI', if_true: pci_ss) diff --git a/hw/pci/pcie_doe.c b/hw/pci/pcie_doe.c new file mode 100644 index 0000000000..2210f86968 --- /dev/null +++ b/hw/pci/pcie_doe.c @@ -0,0 +1,367 @@ +/* + * PCIe Data Object Exchange + * + * Copyright (C) 2021 Avery Design Systems, Inc. + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "qemu/log.h" +#include "qemu/error-report.h" +#include "qapi/error.h" +#include "qemu/range.h" +#include "hw/pci/pci.h" +#include "hw/pci/pcie.h" +#include "hw/pci/pcie_doe.h" +#include "hw/pci/msi.h" +#include "hw/pci/msix.h" + +#define DWORD_BYTE 4 + +typedef struct DoeDiscoveryReq { + DOEHeader header; + uint8_t index; + uint8_t reserved[3]; +} QEMU_PACKED DoeDiscoveryReq; + +typedef struct DoeDiscoveryRsp { + DOEHeader header; + uint16_t vendor_id; + uint8_t data_obj_type; + uint8_t next_index; +} QEMU_PACKED DoeDiscoveryRsp; + +static bool pcie_doe_discovery(DOECap *doe_cap) +{ + DoeDiscoveryReq *req = pcie_doe_get_write_mbox_ptr(doe_cap); + DoeDiscoveryRsp rsp; + uint8_t index = req->index; + DOEProtocol *prot; + + /* Discard request if length does not match DoeDiscoveryReq */ + if (pcie_doe_get_obj_len(req) < + DIV_ROUND_UP(sizeof(DoeDiscoveryReq), DWORD_BYTE)) { + return false; + } + + rsp.header = (DOEHeader) { + .vendor_id = PCI_VENDOR_ID_PCI_SIG, + .data_obj_type = PCI_SIG_DOE_DISCOVERY, + .length = DIV_ROUND_UP(sizeof(DoeDiscoveryRsp), DWORD_BYTE), + }; + + /* Point to the requested protocol, index 0 must be Discovery */ + if (index == 0) { + rsp.vendor_id = PCI_VENDOR_ID_PCI_SIG; + rsp.data_obj_type = PCI_SIG_DOE_DISCOVERY; + } else { + if (index < doe_cap->protocol_num) { + prot = &doe_cap->protocols[index - 1]; + rsp.vendor_id = prot->vendor_id; + rsp.data_obj_type = prot->data_obj_type; + } else { + rsp.vendor_id = 0xFFFF; + rsp.data_obj_type = 0xFF; + } + } + + if (index + 1 == doe_cap->protocol_num) { + rsp.next_index = 0; + } else { + rsp.next_index = index + 1; + } + + pcie_doe_set_rsp(doe_cap, &rsp); + + return true; +} + +static void pcie_doe_reset_mbox(DOECap *st) +{ + st->read_mbox_idx = 0; + st->read_mbox_len = 0; + st->write_mbox_len = 0; + + memset(st->read_mbox, 0, PCI_DOE_DW_SIZE_MAX * DWORD_BYTE); + memset(st->write_mbox, 0, PCI_DOE_DW_SIZE_MAX * DWORD_BYTE); +} + +void pcie_doe_init(PCIDevice *dev, DOECap *doe_cap, uint16_t offset, + DOEProtocol *protocols, bool intr, uint16_t vec) +{ + pcie_add_capability(dev, PCI_EXT_CAP_ID_DOE, 0x1, offset, + PCI_DOE_SIZEOF); + + doe_cap->pdev = dev; + doe_cap->offset = offset; + + if (intr && (msi_present(dev) || msix_present(dev))) { + doe_cap->cap.intr = intr; + doe_cap->cap.vec = vec; + } + + doe_cap->write_mbox = g_malloc0(PCI_DOE_DW_SIZE_MAX * DWORD_BYTE); + doe_cap->read_mbox = g_malloc0(PCI_DOE_DW_SIZE_MAX * DWORD_BYTE); + + pcie_doe_reset_mbox(doe_cap); + + doe_cap->protocols = protocols; + for (; protocols->vendor_id; protocols++) { + doe_cap->protocol_num++; + } + assert(doe_cap->protocol_num < PCI_DOE_PROTOCOL_NUM_MAX); + + /* Increment to allow for the discovery protocol */ + doe_cap->protocol_num++; +} + +void pcie_doe_fini(DOECap *doe_cap) +{ + g_free(doe_cap->read_mbox); + g_free(doe_cap->write_mbox); + g_free(doe_cap); +} + +uint32_t pcie_doe_build_protocol(DOEProtocol *p) +{ + return DATA_OBJ_BUILD_HEADER1(p->vendor_id, p->data_obj_type); +} + +void *pcie_doe_get_write_mbox_ptr(DOECap *doe_cap) +{ + return doe_cap->write_mbox; +} + +/* + * Copy the response to read mailbox buffer + * This might be called in self-defined handle_request() if a DOE response is + * required in the corresponding protocol + */ +void pcie_doe_set_rsp(DOECap *doe_cap, void *rsp) +{ + uint32_t len = pcie_doe_get_obj_len(rsp); + + memcpy(doe_cap->read_mbox + doe_cap->read_mbox_len, rsp, len * DWORD_BYTE); + doe_cap->read_mbox_len += len; +} + +uint32_t pcie_doe_get_obj_len(void *obj) +{ + uint32_t len; + + if (!obj) { + return 0; + } + + /* Only lower 18 bits are valid */ + len = DATA_OBJ_LEN_MASK(((DOEHeader *)obj)->length); + + /* PCIe r6.0 Table 6.29: a value of 00000h indicates 2^18 DW */ + return (len) ? len : PCI_DOE_DW_SIZE_MAX; +} + +static void pcie_doe_irq_assert(DOECap *doe_cap) +{ + PCIDevice *dev = doe_cap->pdev; + + if (doe_cap->cap.intr && doe_cap->ctrl.intr) { + if (doe_cap->status.intr) { + return; + } + doe_cap->status.intr = 1; + + if (msix_enabled(dev)) { + msix_notify(dev, doe_cap->cap.vec); + } else if (msi_enabled(dev)) { + msi_notify(dev, doe_cap->cap.vec); + } + } +} + +static void pcie_doe_set_ready(DOECap *doe_cap, bool rdy) +{ + doe_cap->status.ready = rdy; + + if (rdy) { + pcie_doe_irq_assert(doe_cap); + } +} + +static void pcie_doe_set_error(DOECap *doe_cap, bool err) +{ + doe_cap->status.error = err; + + if (err) { + pcie_doe_irq_assert(doe_cap); + } +} + +/* + * Check incoming request in write_mbox for protocol format + */ +static void pcie_doe_prepare_rsp(DOECap *doe_cap) +{ + bool success = false; + int p; + bool (*handle_request)(DOECap *) = NULL; + + if (doe_cap->status.error) { + return; + } + + if (doe_cap->write_mbox[0] == + DATA_OBJ_BUILD_HEADER1(PCI_VENDOR_ID_PCI_SIG, PCI_SIG_DOE_DISCOVERY)) { + handle_request = pcie_doe_discovery; + } else { + for (p = 0; p < doe_cap->protocol_num - 1; p++) { + if (doe_cap->write_mbox[0] == + pcie_doe_build_protocol(&doe_cap->protocols[p])) { + handle_request = doe_cap->protocols[p].handle_request; + break; + } + } + } + + /* + * PCIe r6 DOE 6.30.1: + * If the number of DW transferred does not match the + * indicated Length for a data object, then the + * data object must be silently discarded. + */ + if (handle_request && (doe_cap->write_mbox_len == + pcie_doe_get_obj_len(pcie_doe_get_write_mbox_ptr(doe_cap)))) { + success = handle_request(doe_cap); + } + + if (success) { + pcie_doe_set_ready(doe_cap, 1); + } else { + pcie_doe_reset_mbox(doe_cap); + } +} + +/* + * Read from DOE config space. + * Return false if the address not within DOE_CAP range. + */ +bool pcie_doe_read_config(DOECap *doe_cap, uint32_t addr, int size, + uint32_t *buf) +{ + uint32_t shift; + uint16_t doe_offset = doe_cap->offset; + + if (!range_covers_byte(doe_offset + PCI_EXP_DOE_CAP, + PCI_DOE_SIZEOF - 4, addr)) { + return false; + } + + addr -= doe_offset; + *buf = 0; + + if (range_covers_byte(PCI_EXP_DOE_CAP, DWORD_BYTE, addr)) { + *buf = FIELD_DP32(*buf, PCI_DOE_CAP_REG, INTR_SUPP, + doe_cap->cap.intr); + *buf = FIELD_DP32(*buf, PCI_DOE_CAP_REG, DOE_INTR_MSG_NUM, + doe_cap->cap.vec); + } else if (range_covers_byte(PCI_EXP_DOE_CTRL, DWORD_BYTE, addr)) { + /* Must return ABORT=0 and GO=0 */ + *buf = FIELD_DP32(*buf, PCI_DOE_CAP_CONTROL, DOE_INTR_EN, + doe_cap->ctrl.intr); + } else if (range_covers_byte(PCI_EXP_DOE_STATUS, DWORD_BYTE, addr)) { + *buf = FIELD_DP32(*buf, PCI_DOE_CAP_STATUS, DOE_BUSY, + doe_cap->status.busy); + *buf = FIELD_DP32(*buf, PCI_DOE_CAP_STATUS, DOE_INTR_STATUS, + doe_cap->status.intr); + *buf = FIELD_DP32(*buf, PCI_DOE_CAP_STATUS, DOE_ERROR, + doe_cap->status.error); + *buf = FIELD_DP32(*buf, PCI_DOE_CAP_STATUS, DATA_OBJ_RDY, + doe_cap->status.ready); + /* Mailbox should be DW accessed */ + } else if (addr == PCI_EXP_DOE_RD_DATA_MBOX && size == DWORD_BYTE) { + if (doe_cap->status.ready && !doe_cap->status.error) { + *buf = doe_cap->read_mbox[doe_cap->read_mbox_idx]; + } + } + + /* Process Alignment */ + shift = addr % DWORD_BYTE; + *buf = extract32(*buf, shift * 8, size * 8); + + return true; +} + +/* + * Write to DOE config space. + * Return if the address not within DOE_CAP range or receives an abort + */ +void pcie_doe_write_config(DOECap *doe_cap, + uint32_t addr, uint32_t val, int size) +{ + uint16_t doe_offset = doe_cap->offset; + uint32_t shift; + + if (!range_covers_byte(doe_offset + PCI_EXP_DOE_CAP, + PCI_DOE_SIZEOF - 4, addr)) { + return; + } + + /* Process Alignment */ + shift = addr % DWORD_BYTE; + addr -= (doe_offset + shift); + val = deposit32(val, shift * 8, size * 8, val); + + switch (addr) { + case PCI_EXP_DOE_CTRL: + if (FIELD_EX32(val, PCI_DOE_CAP_CONTROL, DOE_ABORT)) { + pcie_doe_set_ready(doe_cap, 0); + pcie_doe_set_error(doe_cap, 0); + pcie_doe_reset_mbox(doe_cap); + return; + } + + if (FIELD_EX32(val, PCI_DOE_CAP_CONTROL, DOE_GO)) { + pcie_doe_prepare_rsp(doe_cap); + } + + if (FIELD_EX32(val, PCI_DOE_CAP_CONTROL, DOE_INTR_EN)) { + doe_cap->ctrl.intr = 1; + /* Clear interrupt bit located within the first byte */ + } else if (shift == 0) { + doe_cap->ctrl.intr = 0; + } + break; + case PCI_EXP_DOE_STATUS: + if (FIELD_EX32(val, PCI_DOE_CAP_STATUS, DOE_INTR_STATUS)) { + doe_cap->status.intr = 0; + } + break; + case PCI_EXP_DOE_RD_DATA_MBOX: + /* Mailbox should be DW accessed */ + if (size != DWORD_BYTE) { + return; + } + doe_cap->read_mbox_idx++; + if (doe_cap->read_mbox_idx == doe_cap->read_mbox_len) { + pcie_doe_reset_mbox(doe_cap); + pcie_doe_set_ready(doe_cap, 0); + } else if (doe_cap->read_mbox_idx > doe_cap->read_mbox_len) { + /* Underflow */ + pcie_doe_set_error(doe_cap, 1); + } + break; + case PCI_EXP_DOE_WR_DATA_MBOX: + /* Mailbox should be DW accessed */ + if (size != DWORD_BYTE) { + return; + } + doe_cap->write_mbox[doe_cap->write_mbox_len] = val; + doe_cap->write_mbox_len++; + break; + case PCI_EXP_DOE_CAP: + /* fallthrough */ + default: + break; + } +} diff --git a/include/hw/pci/pci_ids.h b/include/hw/pci/pci_ids.h index d5ddea558b..bc9f834fd1 100644 --- a/include/hw/pci/pci_ids.h +++ b/include/hw/pci/pci_ids.h @@ -157,6 +157,9 @@ /* Vendors and devices. Sort key: vendor first, device next. */ +/* Ref: PCIe r6.0 Table 6-32 */ +#define PCI_VENDOR_ID_PCI_SIG 0x0001 + #define PCI_VENDOR_ID_LSI_LOGIC 0x1000 #define PCI_DEVICE_ID_LSI_53C810 0x0001 #define PCI_DEVICE_ID_LSI_53C895A 0x0012 diff --git a/include/hw/pci/pcie.h b/include/hw/pci/pcie.h index 798a262a0a..698d3de851 100644 --- a/include/hw/pci/pcie.h +++ b/include/hw/pci/pcie.h @@ -26,6 +26,7 @@ #include "hw/pci/pcie_aer.h" #include "hw/pci/pcie_sriov.h" #include "hw/hotplug.h" +#include "hw/pci/pcie_doe.h" typedef enum { /* for attention and power indicator */ diff --git a/include/hw/pci/pcie_doe.h b/include/hw/pci/pcie_doe.h new file mode 100644 index 0000000000..ba4d8b03bd --- /dev/null +++ b/include/hw/pci/pcie_doe.h @@ -0,0 +1,123 @@ +/* + * PCIe Data Object Exchange + * + * Copyright (C) 2021 Avery Design Systems, Inc. + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#ifndef PCIE_DOE_H +#define PCIE_DOE_H + +#include "qemu/range.h" +#include "qemu/typedefs.h" +#include "hw/register.h" + +/* + * Reference: + * PCIe r6.0 - 7.9.24 Data Object Exchange Extended Capability + */ +/* Capabilities Register - r6.0 7.9.24.2 */ +#define PCI_EXP_DOE_CAP 0x04 +REG32(PCI_DOE_CAP_REG, 0) + FIELD(PCI_DOE_CAP_REG, INTR_SUPP, 0, 1) + FIELD(PCI_DOE_CAP_REG, DOE_INTR_MSG_NUM, 1, 11) + +/* Control Register - r6.0 7.9.24.3 */ +#define PCI_EXP_DOE_CTRL 0x08 +REG32(PCI_DOE_CAP_CONTROL, 0) + FIELD(PCI_DOE_CAP_CONTROL, DOE_ABORT, 0, 1) + FIELD(PCI_DOE_CAP_CONTROL, DOE_INTR_EN, 1, 1) + FIELD(PCI_DOE_CAP_CONTROL, DOE_GO, 31, 1) + +/* Status Register - r6.0 7.9.24.4 */ +#define PCI_EXP_DOE_STATUS 0x0c +REG32(PCI_DOE_CAP_STATUS, 0) + FIELD(PCI_DOE_CAP_STATUS, DOE_BUSY, 0, 1) + FIELD(PCI_DOE_CAP_STATUS, DOE_INTR_STATUS, 1, 1) + FIELD(PCI_DOE_CAP_STATUS, DOE_ERROR, 2, 1) + FIELD(PCI_DOE_CAP_STATUS, DATA_OBJ_RDY, 31, 1) + +/* Write Data Mailbox Register - r6.0 7.9.24.5 */ +#define PCI_EXP_DOE_WR_DATA_MBOX 0x10 + +/* Read Data Mailbox Register - 7.9.xx.6 */ +#define PCI_EXP_DOE_RD_DATA_MBOX 0x14 + +/* PCI-SIG defined Data Object Types - r6.0 Table 6-32 */ +#define PCI_SIG_DOE_DISCOVERY 0x00 + +#define PCI_DOE_DW_SIZE_MAX (1 << 18) +#define PCI_DOE_PROTOCOL_NUM_MAX 256 + +#define DATA_OBJ_BUILD_HEADER1(v, p) (((p) << 16) | (v)) +#define DATA_OBJ_LEN_MASK(len) ((len) & (PCI_DOE_DW_SIZE_MAX - 1)) + +typedef struct DOEHeader DOEHeader; +typedef struct DOEProtocol DOEProtocol; +typedef struct DOECap DOECap; + +struct DOEHeader { + uint16_t vendor_id; + uint8_t data_obj_type; + uint8_t reserved; + uint32_t length; +} QEMU_PACKED; + +/* Protocol infos and rsp function callback */ +struct DOEProtocol { + uint16_t vendor_id; + uint8_t data_obj_type; + bool (*handle_request)(DOECap *); +}; + +struct DOECap { + /* Owner */ + PCIDevice *pdev; + + uint16_t offset; + + struct { + bool intr; + uint16_t vec; + } cap; + + struct { + bool abort; + bool intr; + bool go; + } ctrl; + + struct { + bool busy; + bool intr; + bool error; + bool ready; + } status; + + uint32_t *write_mbox; + uint32_t *read_mbox; + + /* Mailbox position indicator */ + uint32_t read_mbox_idx; + uint32_t read_mbox_len; + uint32_t write_mbox_len; + + /* Protocols and its callback response */ + DOEProtocol *protocols; + uint16_t protocol_num; +}; + +void pcie_doe_init(PCIDevice *pdev, DOECap *doe_cap, uint16_t offset, + DOEProtocol *protocols, bool intr, uint16_t vec); +void pcie_doe_fini(DOECap *doe_cap); +bool pcie_doe_read_config(DOECap *doe_cap, uint32_t addr, int size, + uint32_t *buf); +void pcie_doe_write_config(DOECap *doe_cap, uint32_t addr, + uint32_t val, int size); +uint32_t pcie_doe_build_protocol(DOEProtocol *p); +void *pcie_doe_get_write_mbox_ptr(DOECap *doe_cap); +void pcie_doe_set_rsp(DOECap *doe_cap, void *rsp); +uint32_t pcie_doe_get_obj_len(void *obj); +#endif /* PCIE_DOE_H */ diff --git a/include/hw/pci/pcie_regs.h b/include/hw/pci/pcie_regs.h index 1db86b0ec4..963dc2e170 100644 --- a/include/hw/pci/pcie_regs.h +++ b/include/hw/pci/pcie_regs.h @@ -179,4 +179,8 @@ typedef enum PCIExpLinkWidth { #define PCI_ACS_VER 0x1 #define PCI_ACS_SIZEOF 8 +/* DOE Capability Register Fields */ +#define PCI_DOE_VER 0x1 +#define PCI_DOE_SIZEOF 24 + #endif /* QEMU_PCIE_REGS_H */ -- cgit v1.2.3-55-g7522 From aba578bdace5303a441f8a37aad781b5cb06f38c Mon Sep 17 00:00:00 2001 From: Huai-Cheng Kuo Date: Fri, 14 Oct 2022 16:10:43 +0100 Subject: hw/cxl/cdat: CXL CDAT Data Object Exchange implementation The Data Object Exchange implementation of CXL Coherent Device Attribute Table (CDAT). This implementation is referring to "Coherent Device Attribute Table Specification, Rev. 1.03, July. 2022" and "Compute Express Link Specification, Rev. 3.0, July. 2022" This patch adds core support that will be shared by both end-points and switch port emulation. Signed-off-by: Huai-Cheng Kuo Signed-off-by: Chris Browy Signed-off-by: Jonathan Cameron Message-Id: <20221014151045.24781-4-Jonathan.Cameron@huawei.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- hw/cxl/cxl-cdat.c | 224 +++++++++++++++++++++++++++++++++++++++++ hw/cxl/meson.build | 1 + include/hw/cxl/cxl_cdat.h | 165 ++++++++++++++++++++++++++++++ include/hw/cxl/cxl_component.h | 7 ++ include/hw/cxl/cxl_device.h | 3 + include/hw/cxl/cxl_pci.h | 1 + 6 files changed, 401 insertions(+) create mode 100644 hw/cxl/cxl-cdat.c create mode 100644 include/hw/cxl/cxl_cdat.h (limited to 'include') diff --git a/hw/cxl/cxl-cdat.c b/hw/cxl/cxl-cdat.c new file mode 100644 index 0000000000..3653aa56f0 --- /dev/null +++ b/hw/cxl/cxl-cdat.c @@ -0,0 +1,224 @@ +/* + * CXL CDAT Structure + * + * Copyright (C) 2021 Avery Design Systems, Inc. + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#include "qemu/osdep.h" +#include "hw/pci/pci.h" +#include "hw/cxl/cxl.h" +#include "qapi/error.h" +#include "qemu/error-report.h" + +static void cdat_len_check(CDATSubHeader *hdr, Error **errp) +{ + assert(hdr->length); + assert(hdr->reserved == 0); + + switch (hdr->type) { + case CDAT_TYPE_DSMAS: + assert(hdr->length == sizeof(CDATDsmas)); + break; + case CDAT_TYPE_DSLBIS: + assert(hdr->length == sizeof(CDATDslbis)); + break; + case CDAT_TYPE_DSMSCIS: + assert(hdr->length == sizeof(CDATDsmscis)); + break; + case CDAT_TYPE_DSIS: + assert(hdr->length == sizeof(CDATDsis)); + break; + case CDAT_TYPE_DSEMTS: + assert(hdr->length == sizeof(CDATDsemts)); + break; + case CDAT_TYPE_SSLBIS: + assert(hdr->length >= sizeof(CDATSslbisHeader)); + assert((hdr->length - sizeof(CDATSslbisHeader)) % + sizeof(CDATSslbe) == 0); + break; + default: + error_setg(errp, "Type %d is reserved", hdr->type); + } +} + +static void ct3_build_cdat(CDATObject *cdat, Error **errp) +{ + g_autofree CDATTableHeader *cdat_header = NULL; + g_autofree CDATEntry *cdat_st = NULL; + uint8_t sum = 0; + int ent, i; + + /* Use default table if fopen == NULL */ + assert(cdat->build_cdat_table); + + cdat_header = g_malloc0(sizeof(*cdat_header)); + if (!cdat_header) { + error_setg(errp, "Failed to allocate CDAT header"); + return; + } + + cdat->built_buf_len = cdat->build_cdat_table(&cdat->built_buf, cdat->private); + + if (!cdat->built_buf_len) { + /* Build later as not all data available yet */ + cdat->to_update = true; + return; + } + cdat->to_update = false; + + cdat_st = g_malloc0(sizeof(*cdat_st) * (cdat->built_buf_len + 1)); + if (!cdat_st) { + error_setg(errp, "Failed to allocate CDAT entry array"); + return; + } + + /* Entry 0 for CDAT header, starts with Entry 1 */ + for (ent = 1; ent < cdat->built_buf_len + 1; ent++) { + CDATSubHeader *hdr = cdat->built_buf[ent - 1]; + uint8_t *buf = (uint8_t *)cdat->built_buf[ent - 1]; + + cdat_st[ent].base = hdr; + cdat_st[ent].length = hdr->length; + + cdat_header->length += hdr->length; + for (i = 0; i < hdr->length; i++) { + sum += buf[i]; + } + } + + /* CDAT header */ + cdat_header->revision = CXL_CDAT_REV; + /* For now, no runtime updates */ + cdat_header->sequence = 0; + cdat_header->length += sizeof(CDATTableHeader); + sum += cdat_header->revision + cdat_header->sequence + + cdat_header->length; + /* Sum of all bytes including checksum must be 0 */ + cdat_header->checksum = ~sum + 1; + + cdat_st[0].base = g_steal_pointer(&cdat_header); + cdat_st[0].length = sizeof(*cdat_header); + cdat->entry_len = 1 + cdat->built_buf_len; + cdat->entry = g_steal_pointer(&cdat_st); +} + +static void ct3_load_cdat(CDATObject *cdat, Error **errp) +{ + g_autofree CDATEntry *cdat_st = NULL; + uint8_t sum = 0; + int num_ent; + int i = 0, ent = 1, file_size = 0; + CDATSubHeader *hdr; + FILE *fp = NULL; + + /* Read CDAT file and create its cache */ + fp = fopen(cdat->filename, "r"); + if (!fp) { + error_setg(errp, "CDAT: Unable to open file"); + return; + } + + fseek(fp, 0, SEEK_END); + file_size = ftell(fp); + fseek(fp, 0, SEEK_SET); + cdat->buf = g_malloc0(file_size); + + if (fread(cdat->buf, file_size, 1, fp) == 0) { + error_setg(errp, "CDAT: File read failed"); + return; + } + + fclose(fp); + + if (file_size < sizeof(CDATTableHeader)) { + error_setg(errp, "CDAT: File too short"); + return; + } + i = sizeof(CDATTableHeader); + num_ent = 1; + while (i < file_size) { + hdr = (CDATSubHeader *)(cdat->buf + i); + cdat_len_check(hdr, errp); + i += hdr->length; + num_ent++; + } + if (i != file_size) { + error_setg(errp, "CDAT: File length missmatch"); + return; + } + + cdat_st = g_malloc0(sizeof(*cdat_st) * num_ent); + if (!cdat_st) { + error_setg(errp, "CDAT: Failed to allocate entry array"); + return; + } + + /* Set CDAT header, Entry = 0 */ + cdat_st[0].base = cdat->buf; + cdat_st[0].length = sizeof(CDATTableHeader); + i = 0; + + while (i < cdat_st[0].length) { + sum += cdat->buf[i++]; + } + + /* Read CDAT structures */ + while (i < file_size) { + hdr = (CDATSubHeader *)(cdat->buf + i); + cdat_len_check(hdr, errp); + + cdat_st[ent].base = hdr; + cdat_st[ent].length = hdr->length; + + while (cdat->buf + i < + (uint8_t *)cdat_st[ent].base + cdat_st[ent].length) { + assert(i < file_size); + sum += cdat->buf[i++]; + } + + ent++; + } + + if (sum != 0) { + warn_report("CDAT: Found checksum mismatch in %s", cdat->filename); + } + cdat->entry_len = num_ent; + cdat->entry = g_steal_pointer(&cdat_st); +} + +void cxl_doe_cdat_init(CXLComponentState *cxl_cstate, Error **errp) +{ + CDATObject *cdat = &cxl_cstate->cdat; + + if (cdat->filename) { + ct3_load_cdat(cdat, errp); + } else { + ct3_build_cdat(cdat, errp); + } +} + +void cxl_doe_cdat_update(CXLComponentState *cxl_cstate, Error **errp) +{ + CDATObject *cdat = &cxl_cstate->cdat; + + if (cdat->to_update) { + ct3_build_cdat(cdat, errp); + } +} + +void cxl_doe_cdat_release(CXLComponentState *cxl_cstate) +{ + CDATObject *cdat = &cxl_cstate->cdat; + + free(cdat->entry); + if (cdat->built_buf) { + cdat->free_cdat_table(cdat->built_buf, cdat->built_buf_len, + cdat->private); + } + if (cdat->buf) { + free(cdat->buf); + } +} diff --git a/hw/cxl/meson.build b/hw/cxl/meson.build index f117b99949..cfa95ffd40 100644 --- a/hw/cxl/meson.build +++ b/hw/cxl/meson.build @@ -4,6 +4,7 @@ softmmu_ss.add(when: 'CONFIG_CXL', 'cxl-device-utils.c', 'cxl-mailbox-utils.c', 'cxl-host.c', + 'cxl-cdat.c', ), if_false: files( 'cxl-host-stubs.c', diff --git a/include/hw/cxl/cxl_cdat.h b/include/hw/cxl/cxl_cdat.h new file mode 100644 index 0000000000..52c232e912 --- /dev/null +++ b/include/hw/cxl/cxl_cdat.h @@ -0,0 +1,165 @@ +/* + * CXL CDAT Structure + * + * Copyright (C) 2021 Avery Design Systems, Inc. + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#ifndef CXL_CDAT_H +#define CXL_CDAT_H + +#include "hw/cxl/cxl_pci.h" + +/* + * Reference: + * Coherent Device Attribute Table (CDAT) Specification, Rev. 1.03, July. 2022 + * Compute Express Link (CXL) Specification, Rev. 3.0, Aug. 2022 + */ +/* Table Access DOE - CXL r3.0 8.1.11 */ +#define CXL_DOE_TABLE_ACCESS 2 +#define CXL_DOE_PROTOCOL_CDAT ((CXL_DOE_TABLE_ACCESS << 16) | CXL_VENDOR_ID) + +/* Read Entry - CXL r3.0 8.1.11.1 */ +#define CXL_DOE_TAB_TYPE_CDAT 0 +#define CXL_DOE_TAB_ENT_MAX 0xFFFF + +/* Read Entry Request - CXL r3.0 8.1.11.1 Table 8-13 */ +#define CXL_DOE_TAB_REQ 0 +typedef struct CDATReq { + DOEHeader header; + uint8_t req_code; + uint8_t table_type; + uint16_t entry_handle; +} QEMU_PACKED CDATReq; + +/* Read Entry Response - CXL r3.0 8.1.11.1 Table 8-14 */ +#define CXL_DOE_TAB_RSP 0 +typedef struct CDATRsp { + DOEHeader header; + uint8_t rsp_code; + uint8_t table_type; + uint16_t entry_handle; +} QEMU_PACKED CDATRsp; + +/* CDAT Table Format - CDAT Table 1 */ +#define CXL_CDAT_REV 2 +typedef struct CDATTableHeader { + uint32_t length; + uint8_t revision; + uint8_t checksum; + uint8_t reserved[6]; + uint32_t sequence; +} QEMU_PACKED CDATTableHeader; + +/* CDAT Structure Types - CDAT Table 2 */ +typedef enum { + CDAT_TYPE_DSMAS = 0, + CDAT_TYPE_DSLBIS = 1, + CDAT_TYPE_DSMSCIS = 2, + CDAT_TYPE_DSIS = 3, + CDAT_TYPE_DSEMTS = 4, + CDAT_TYPE_SSLBIS = 5, +} CDATType; + +typedef struct CDATSubHeader { + uint8_t type; + uint8_t reserved; + uint16_t length; +} CDATSubHeader; + +/* Device Scoped Memory Affinity Structure - CDAT Table 3 */ +typedef struct CDATDsmas { + CDATSubHeader header; + uint8_t DSMADhandle; + uint8_t flags; +#define CDAT_DSMAS_FLAG_NV (1 << 2) +#define CDAT_DSMAS_FLAG_SHAREABLE (1 << 3) +#define CDAT_DSMAS_FLAG_HW_COHERENT (1 << 4) +#define CDAT_DSMAS_FLAG_DYNAMIC_CAP (1 << 5) + uint16_t reserved; + uint64_t DPA_base; + uint64_t DPA_length; +} QEMU_PACKED CDATDsmas; + +/* Device Scoped Latency and Bandwidth Information Structure - CDAT Table 5 */ +typedef struct CDATDslbis { + CDATSubHeader header; + uint8_t handle; + /* Definitions of these fields refer directly to HMAT fields */ + uint8_t flags; + uint8_t data_type; + uint8_t reserved; + uint64_t entry_base_unit; + uint16_t entry[3]; + uint16_t reserved2; +} QEMU_PACKED CDATDslbis; + +/* Device Scoped Memory Side Cache Information Structure - CDAT Table 6 */ +typedef struct CDATDsmscis { + CDATSubHeader header; + uint8_t DSMAS_handle; + uint8_t reserved[3]; + uint64_t memory_side_cache_size; + uint32_t cache_attributes; +} QEMU_PACKED CDATDsmscis; + +/* Device Scoped Initiator Structure - CDAT Table 7 */ +typedef struct CDATDsis { + CDATSubHeader header; + uint8_t flags; + uint8_t handle; + uint16_t reserved; +} QEMU_PACKED CDATDsis; + +/* Device Scoped EFI Memory Type Structure - CDAT Table 8 */ +typedef struct CDATDsemts { + CDATSubHeader header; + uint8_t DSMAS_handle; + uint8_t EFI_memory_type_attr; + uint16_t reserved; + uint64_t DPA_offset; + uint64_t DPA_length; +} QEMU_PACKED CDATDsemts; + +/* Switch Scoped Latency and Bandwidth Information Structure - CDAT Table 9 */ +typedef struct CDATSslbisHeader { + CDATSubHeader header; + uint8_t data_type; + uint8_t reserved[3]; + uint64_t entry_base_unit; +} QEMU_PACKED CDATSslbisHeader; + +/* Switch Scoped Latency and Bandwidth Entry - CDAT Table 10 */ +typedef struct CDATSslbe { + uint16_t port_x_id; + uint16_t port_y_id; + uint16_t latency_bandwidth; + uint16_t reserved; +} QEMU_PACKED CDATSslbe; + +typedef struct CDATSslbis { + CDATSslbisHeader sslbis_header; + CDATSslbe sslbe[]; +} QEMU_PACKED CDATSslbis; + +typedef struct CDATEntry { + void *base; + uint32_t length; +} CDATEntry; + +typedef struct CDATObject { + CDATEntry *entry; + int entry_len; + + int (*build_cdat_table)(CDATSubHeader ***cdat_table, void *priv); + void (*free_cdat_table)(CDATSubHeader **cdat_table, int num, void *priv); + bool to_update; + void *private; + char *filename; + uint8_t *buf; + struct CDATSubHeader **built_buf; + int built_buf_len; +} CDATObject; +#endif /* CXL_CDAT_H */ diff --git a/include/hw/cxl/cxl_component.h b/include/hw/cxl/cxl_component.h index 94ec2f07d7..34075cfb72 100644 --- a/include/hw/cxl/cxl_component.h +++ b/include/hw/cxl/cxl_component.h @@ -19,6 +19,7 @@ #include "qemu/range.h" #include "qemu/typedefs.h" #include "hw/register.h" +#include "qapi/error.h" enum reg_type { CXL2_DEVICE, @@ -184,6 +185,8 @@ typedef struct cxl_component { struct PCIDevice *pdev; }; }; + + CDATObject cdat; } CXLComponentState; void cxl_component_register_block_init(Object *obj, @@ -220,4 +223,8 @@ static inline hwaddr cxl_decode_ig(int ig) CXLComponentState *cxl_get_hb_cstate(PCIHostState *hb); +void cxl_doe_cdat_init(CXLComponentState *cxl_cstate, Error **errp); +void cxl_doe_cdat_release(CXLComponentState *cxl_cstate); +void cxl_doe_cdat_update(CXLComponentState *cxl_cstate, Error **errp); + #endif diff --git a/include/hw/cxl/cxl_device.h b/include/hw/cxl/cxl_device.h index e4d221cdb3..449b0edfe9 100644 --- a/include/hw/cxl/cxl_device.h +++ b/include/hw/cxl/cxl_device.h @@ -243,6 +243,9 @@ struct CXLType3Dev { AddressSpace hostmem_as; CXLComponentState cxl_cstate; CXLDeviceState cxl_dstate; + + /* DOE */ + DOECap doe_cdat; }; #define TYPE_CXL_TYPE3 "cxl-type3" diff --git a/include/hw/cxl/cxl_pci.h b/include/hw/cxl/cxl_pci.h index 01cf002096..3cb79eca1e 100644 --- a/include/hw/cxl/cxl_pci.h +++ b/include/hw/cxl/cxl_pci.h @@ -13,6 +13,7 @@ #include "qemu/compiler.h" #include "hw/pci/pci.h" #include "hw/pci/pcie.h" +#include "hw/cxl/cxl_cdat.h" #define CXL_VENDOR_ID 0x1e98 -- cgit v1.2.3-55-g7522 From 882877fc359d24e1563065c5c3887096317ca1ae Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Fri, 14 Oct 2022 16:10:45 +0100 Subject: hw/pci-bridge/cxl-upstream: Add a CDAT table access DOE This Data Object Exchange Mailbox allows software to query the latency and bandwidth between ports on the switch. For now only provide information on routes between the upstream port and each downstream port (not p2p). Signed-off-by: Jonathan Cameron -- Changes since v8: Mostly to match the type 3 equivalent - Move enum out of function and give it a more descriptive namespace. Message-Id: <20221014151045.24781-6-Jonathan.Cameron@huawei.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- hw/pci-bridge/cxl_upstream.c | 195 ++++++++++++++++++++++++++++++++++++++++++- include/hw/cxl/cxl_cdat.h | 1 + 2 files changed, 195 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/hw/pci-bridge/cxl_upstream.c b/hw/pci-bridge/cxl_upstream.c index a83a3e81e4..9b8b57df9d 100644 --- a/hw/pci-bridge/cxl_upstream.c +++ b/hw/pci-bridge/cxl_upstream.c @@ -10,11 +10,12 @@ #include "qemu/osdep.h" #include "qemu/log.h" +#include "hw/qdev-properties.h" #include "hw/pci/msi.h" #include "hw/pci/pcie.h" #include "hw/pci/pcie_port.h" -#define CXL_UPSTREAM_PORT_MSI_NR_VECTOR 1 +#define CXL_UPSTREAM_PORT_MSI_NR_VECTOR 2 #define CXL_UPSTREAM_PORT_MSI_OFFSET 0x70 #define CXL_UPSTREAM_PORT_PCIE_CAP_OFFSET 0x90 @@ -28,6 +29,7 @@ typedef struct CXLUpstreamPort { /*< public >*/ CXLComponentState cxl_cstate; + DOECap doe_cdat; } CXLUpstreamPort; CXLComponentState *cxl_usp_to_cstate(CXLUpstreamPort *usp) @@ -60,6 +62,9 @@ static void cxl_usp_dvsec_write_config(PCIDevice *dev, uint32_t addr, static void cxl_usp_write_config(PCIDevice *d, uint32_t address, uint32_t val, int len) { + CXLUpstreamPort *usp = CXL_USP(d); + + pcie_doe_write_config(&usp->doe_cdat, address, val, len); pci_bridge_write_config(d, address, val, len); pcie_cap_flr_write_config(d, address, val, len); pcie_aer_write_config(d, address, val, len); @@ -67,6 +72,18 @@ static void cxl_usp_write_config(PCIDevice *d, uint32_t address, cxl_usp_dvsec_write_config(d, address, val, len); } +static uint32_t cxl_usp_read_config(PCIDevice *d, uint32_t address, int len) +{ + CXLUpstreamPort *usp = CXL_USP(d); + uint32_t val; + + if (pcie_doe_read_config(&usp->doe_cdat, address, len, &val)) { + return val; + } + + return pci_default_read_config(d, address, len); +} + static void latch_registers(CXLUpstreamPort *usp) { uint32_t *reg_state = usp->cxl_cstate.crb.cache_mem_registers; @@ -119,6 +136,167 @@ static void build_dvsecs(CXLComponentState *cxl) REG_LOC_DVSEC_REVID, dvsec); } +static bool cxl_doe_cdat_rsp(DOECap *doe_cap) +{ + CDATObject *cdat = &CXL_USP(doe_cap->pdev)->cxl_cstate.cdat; + uint16_t ent; + void *base; + uint32_t len; + CDATReq *req = pcie_doe_get_write_mbox_ptr(doe_cap); + CDATRsp rsp; + + cxl_doe_cdat_update(&CXL_USP(doe_cap->pdev)->cxl_cstate, &error_fatal); + assert(cdat->entry_len); + + /* Discard if request length mismatched */ + if (pcie_doe_get_obj_len(req) < + DIV_ROUND_UP(sizeof(CDATReq), sizeof(uint32_t))) { + return false; + } + + ent = req->entry_handle; + base = cdat->entry[ent].base; + len = cdat->entry[ent].length; + + rsp = (CDATRsp) { + .header = { + .vendor_id = CXL_VENDOR_ID, + .data_obj_type = CXL_DOE_TABLE_ACCESS, + .reserved = 0x0, + .length = DIV_ROUND_UP((sizeof(rsp) + len), sizeof(uint32_t)), + }, + .rsp_code = CXL_DOE_TAB_RSP, + .table_type = CXL_DOE_TAB_TYPE_CDAT, + .entry_handle = (ent < cdat->entry_len - 1) ? + ent + 1 : CXL_DOE_TAB_ENT_MAX, + }; + + memcpy(doe_cap->read_mbox, &rsp, sizeof(rsp)); + memcpy(doe_cap->read_mbox + DIV_ROUND_UP(sizeof(rsp), sizeof(uint32_t)), + base, len); + + doe_cap->read_mbox_len += rsp.header.length; + + return true; +} + +static DOEProtocol doe_cdat_prot[] = { + { CXL_VENDOR_ID, CXL_DOE_TABLE_ACCESS, cxl_doe_cdat_rsp }, + { } +}; + +enum { + CXL_USP_CDAT_SSLBIS_LAT, + CXL_USP_CDAT_SSLBIS_BW, + CXL_USP_CDAT_NUM_ENTRIES +}; + +static int build_cdat_table(CDATSubHeader ***cdat_table, void *priv) +{ + g_autofree CDATSslbis *sslbis_latency = NULL; + g_autofree CDATSslbis *sslbis_bandwidth = NULL; + CXLUpstreamPort *us = CXL_USP(priv); + PCIBus *bus = &PCI_BRIDGE(us)->sec_bus; + int devfn, sslbis_size, i; + int count = 0; + uint16_t port_ids[256]; + + for (devfn = 0; devfn < ARRAY_SIZE(bus->devices); devfn++) { + PCIDevice *d = bus->devices[devfn]; + PCIEPort *port; + + if (!d || !pci_is_express(d) || !d->exp.exp_cap) { + continue; + } + + /* + * Whilst the PCI express spec doesn't allow anything other than + * downstream ports on this bus, let us be a little paranoid + */ + if (!object_dynamic_cast(OBJECT(d), TYPE_PCIE_PORT)) { + continue; + } + + port = PCIE_PORT(d); + port_ids[count] = port->port; + count++; + } + + /* May not yet have any ports - try again later */ + if (count == 0) { + return 0; + } + + sslbis_size = sizeof(CDATSslbis) + sizeof(*sslbis_latency->sslbe) * count; + sslbis_latency = g_malloc(sslbis_size); + if (!sslbis_latency) { + return -ENOMEM; + } + *sslbis_latency = (CDATSslbis) { + .sslbis_header = { + .header = { + .type = CDAT_TYPE_SSLBIS, + .length = sslbis_size, + }, + .data_type = HMATLB_DATA_TYPE_ACCESS_LATENCY, + .entry_base_unit = 10000, + }, + }; + + for (i = 0; i < count; i++) { + sslbis_latency->sslbe[i] = (CDATSslbe) { + .port_x_id = CDAT_PORT_ID_USP, + .port_y_id = port_ids[i], + .latency_bandwidth = 15, /* 150ns */ + }; + } + + sslbis_bandwidth = g_malloc(sslbis_size); + if (!sslbis_bandwidth) { + return 0; + } + *sslbis_bandwidth = (CDATSslbis) { + .sslbis_header = { + .header = { + .type = CDAT_TYPE_SSLBIS, + .length = sslbis_size, + }, + .data_type = HMATLB_DATA_TYPE_ACCESS_BANDWIDTH, + .entry_base_unit = 1000, + }, + }; + + for (i = 0; i < count; i++) { + sslbis_bandwidth->sslbe[i] = (CDATSslbe) { + .port_x_id = CDAT_PORT_ID_USP, + .port_y_id = port_ids[i], + .latency_bandwidth = 16, /* 16 GB/s */ + }; + } + + *cdat_table = g_malloc0(sizeof(*cdat_table) * CXL_USP_CDAT_NUM_ENTRIES); + if (!*cdat_table) { + return -ENOMEM; + } + + /* Header always at start of structure */ + (*cdat_table)[CXL_USP_CDAT_SSLBIS_LAT] = g_steal_pointer(&sslbis_latency); + (*cdat_table)[CXL_USP_CDAT_SSLBIS_BW] = g_steal_pointer(&sslbis_bandwidth); + + return CXL_USP_CDAT_NUM_ENTRIES; +} + +static void free_default_cdat_table(CDATSubHeader **cdat_table, int num, + void *priv) +{ + int i; + + for (i = 0; i < num; i++) { + g_free(cdat_table[i]); + } + g_free(cdat_table); +} + static void cxl_usp_realize(PCIDevice *d, Error **errp) { PCIEPort *p = PCIE_PORT(d); @@ -161,6 +339,14 @@ static void cxl_usp_realize(PCIDevice *d, Error **errp) PCI_BASE_ADDRESS_MEM_TYPE_64, component_bar); + pcie_doe_init(d, &usp->doe_cdat, cxl_cstate->dvsec_offset, doe_cdat_prot, + true, 1); + + cxl_cstate->cdat.build_cdat_table = build_cdat_table; + cxl_cstate->cdat.free_cdat_table = free_default_cdat_table; + cxl_cstate->cdat.private = d; + cxl_doe_cdat_init(cxl_cstate, errp); + return; err_cap: @@ -179,6 +365,11 @@ static void cxl_usp_exitfn(PCIDevice *d) pci_bridge_exitfn(d); } +static Property cxl_upstream_props[] = { + DEFINE_PROP_STRING("cdat", CXLUpstreamPort, cxl_cstate.cdat.filename), + DEFINE_PROP_END_OF_LIST() +}; + static void cxl_upstream_class_init(ObjectClass *oc, void *data) { DeviceClass *dc = DEVICE_CLASS(oc); @@ -186,6 +377,7 @@ static void cxl_upstream_class_init(ObjectClass *oc, void *data) k->is_bridge = true; k->config_write = cxl_usp_write_config; + k->config_read = cxl_usp_read_config; k->realize = cxl_usp_realize; k->exit = cxl_usp_exitfn; k->vendor_id = 0x19e5; /* Huawei */ @@ -194,6 +386,7 @@ static void cxl_upstream_class_init(ObjectClass *oc, void *data) set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories); dc->desc = "CXL Switch Upstream Port"; dc->reset = cxl_usp_reset; + device_class_set_props(dc, cxl_upstream_props); } static const TypeInfo cxl_usp_info = { diff --git a/include/hw/cxl/cxl_cdat.h b/include/hw/cxl/cxl_cdat.h index 52c232e912..e9eda00142 100644 --- a/include/hw/cxl/cxl_cdat.h +++ b/include/hw/cxl/cxl_cdat.h @@ -131,6 +131,7 @@ typedef struct CDATSslbisHeader { uint64_t entry_base_unit; } QEMU_PACKED CDATSslbisHeader; +#define CDAT_PORT_ID_USP 0x100 /* Switch Scoped Latency and Bandwidth Entry - CDAT Table 10 */ typedef struct CDATSslbe { uint16_t port_x_id; -- cgit v1.2.3-55-g7522 From b3a8d6f431730858162feb2dfcee85d0f3484992 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 17 Oct 2022 17:25:45 +0800 Subject: virtio: introduce virtio_queue_reset() Introduce a new interface function virtio_queue_reset() to implement reset for vq. Add a new callback to VirtioDeviceClass for queue reset operation for each child device. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20221017092558.111082-3-xuanzhuo@linux.alibaba.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- hw/virtio/virtio.c | 11 +++++++++++ include/hw/virtio/virtio.h | 2 ++ 2 files changed, 13 insertions(+) (limited to 'include') diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c index 6f42fcadd7..cf5f9ca387 100644 --- a/hw/virtio/virtio.c +++ b/hw/virtio/virtio.c @@ -2484,6 +2484,17 @@ static void __virtio_queue_reset(VirtIODevice *vdev, uint32_t i) virtio_virtqueue_reset_region_cache(&vdev->vq[i]); } +void virtio_queue_reset(VirtIODevice *vdev, uint32_t queue_index) +{ + VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); + + if (k->queue_reset) { + k->queue_reset(vdev, queue_index); + } + + __virtio_queue_reset(vdev, queue_index); +} + void virtio_reset(void *opaque) { VirtIODevice *vdev = opaque; diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h index f41b4a7e64..74d76c1dbc 100644 --- a/include/hw/virtio/virtio.h +++ b/include/hw/virtio/virtio.h @@ -148,6 +148,7 @@ struct VirtioDeviceClass { void (*set_config)(VirtIODevice *vdev, const uint8_t *config); void (*reset)(VirtIODevice *vdev); void (*set_status)(VirtIODevice *vdev, uint8_t val); + void (*queue_reset)(VirtIODevice *vdev, uint32_t queue_index); /* For transitional devices, this is a bitmap of features * that are only exposed on the legacy interface but not * the modern one. @@ -286,6 +287,7 @@ int virtio_queue_set_host_notifier_mr(VirtIODevice *vdev, int n, MemoryRegion *mr, bool assign); int virtio_set_status(VirtIODevice *vdev, uint8_t val); void virtio_reset(void *opaque); +void virtio_queue_reset(VirtIODevice *vdev, uint32_t queue_index); void virtio_update_irq(VirtIODevice *vdev); int virtio_set_features(VirtIODevice *vdev, uint64_t val); -- cgit v1.2.3-55-g7522 From 3c37f8b8d1e5f524c9d04d3e1d8125334e93e2eb Mon Sep 17 00:00:00 2001 From: Kangjie Xu Date: Mon, 17 Oct 2022 17:25:46 +0800 Subject: virtio: introduce virtio_queue_enable() Introduce the interface queue_enable() in VirtioDeviceClass and the fucntion virtio_queue_enable() in virtio, it can be called when VIRTIO_PCI_COMMON_Q_ENABLE is written and related virtqueue can be started. It only supports the devices of virtio 1 or later. The not-supported devices can only start the virtqueue when DRIVER_OK. Signed-off-by: Kangjie Xu Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20221017092558.111082-4-xuanzhuo@linux.alibaba.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- hw/virtio/virtio.c | 14 ++++++++++++++ include/hw/virtio/virtio.h | 2 ++ 2 files changed, 16 insertions(+) (limited to 'include') diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c index cf5f9ca387..9683b2e158 100644 --- a/hw/virtio/virtio.c +++ b/hw/virtio/virtio.c @@ -2495,6 +2495,20 @@ void virtio_queue_reset(VirtIODevice *vdev, uint32_t queue_index) __virtio_queue_reset(vdev, queue_index); } +void virtio_queue_enable(VirtIODevice *vdev, uint32_t queue_index) +{ + VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev); + + if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) { + error_report("queue_enable is only suppported in devices of virtio " + "1.0 or later."); + } + + if (k->queue_enable) { + k->queue_enable(vdev, queue_index); + } +} + void virtio_reset(void *opaque) { VirtIODevice *vdev = opaque; diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h index 74d76c1dbc..b00b3fcf31 100644 --- a/include/hw/virtio/virtio.h +++ b/include/hw/virtio/virtio.h @@ -149,6 +149,7 @@ struct VirtioDeviceClass { void (*reset)(VirtIODevice *vdev); void (*set_status)(VirtIODevice *vdev, uint8_t val); void (*queue_reset)(VirtIODevice *vdev, uint32_t queue_index); + void (*queue_enable)(VirtIODevice *vdev, uint32_t queue_index); /* For transitional devices, this is a bitmap of features * that are only exposed on the legacy interface but not * the modern one. @@ -288,6 +289,7 @@ int virtio_queue_set_host_notifier_mr(VirtIODevice *vdev, int n, int virtio_set_status(VirtIODevice *vdev, uint8_t val); void virtio_reset(void *opaque); void virtio_queue_reset(VirtIODevice *vdev, uint32_t queue_index); +void virtio_queue_enable(VirtIODevice *vdev, uint32_t queue_index); void virtio_update_irq(VirtIODevice *vdev); int virtio_set_features(VirtIODevice *vdev, uint64_t val); -- cgit v1.2.3-55-g7522 From 69e1c14aa22284f933a6ea134b96d5cb5a88a94d Mon Sep 17 00:00:00 2001 From: Kangjie Xu Date: Mon, 17 Oct 2022 17:25:47 +0800 Subject: virtio: core: vq reset feature negotation support A a new command line parameter "queue_reset" is added. Meanwhile, the vq reset feature is disabled for pre-7.2 machines. Signed-off-by: Kangjie Xu Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20221017092558.111082-5-xuanzhuo@linux.alibaba.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- hw/core/machine.c | 4 +++- include/hw/virtio/virtio.h | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/hw/core/machine.c b/hw/core/machine.c index aa520e74a8..907fa78ff0 100644 --- a/hw/core/machine.c +++ b/hw/core/machine.c @@ -40,7 +40,9 @@ #include "hw/virtio/virtio-pci.h" #include "qom/object_interfaces.h" -GlobalProperty hw_compat_7_1[] = {}; +GlobalProperty hw_compat_7_1[] = { + { "virtio-device", "queue_reset", "false" }, +}; const size_t hw_compat_7_1_len = G_N_ELEMENTS(hw_compat_7_1); GlobalProperty hw_compat_7_0[] = { diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h index b00b3fcf31..1423dba379 100644 --- a/include/hw/virtio/virtio.h +++ b/include/hw/virtio/virtio.h @@ -313,7 +313,9 @@ typedef struct VirtIORNGConf VirtIORNGConf; DEFINE_PROP_BIT64("iommu_platform", _state, _field, \ VIRTIO_F_IOMMU_PLATFORM, false), \ DEFINE_PROP_BIT64("packed", _state, _field, \ - VIRTIO_F_RING_PACKED, false) + VIRTIO_F_RING_PACKED, false), \ + DEFINE_PROP_BIT64("queue_reset", _state, _field, \ + VIRTIO_F_RING_RESET, true) hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n); bool virtio_queue_enabled_legacy(VirtIODevice *vdev, int n); -- cgit v1.2.3-55-g7522 From 805d782d284cba013707d5edf38b04047b21d4b5 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 17 Oct 2022 17:25:48 +0800 Subject: virtio-pci: support queue reset PCI devices support vq reset. Based on this function, the driver can adjust the size of the ring, and quickly recycle the buffer in the ring. The migration of the virtio devices will not happen during a reset operation. This is becuase the global iothread lock is held. Migration thread also needs the lock. As a result, when migration of virtio devices starts, the 'reset' status of VirtIOPCIQueue will always be 0. Thus, we do not need to add it in vmstate_virtio_pci_modern_queue_state. Signed-off-by: Xuan Zhuo Signed-off-by: Kangjie Xu Acked-by: Jason Wang Message-Id: <20221017092558.111082-6-xuanzhuo@linux.alibaba.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- hw/virtio/virtio-pci.c | 15 +++++++++++++++ include/hw/virtio/virtio-pci.h | 5 +++++ 2 files changed, 20 insertions(+) (limited to 'include') diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c index 34db51e241..d4f2ffe986 100644 --- a/hw/virtio/virtio-pci.c +++ b/hw/virtio/virtio-pci.c @@ -1251,6 +1251,9 @@ static uint64_t virtio_pci_common_read(void *opaque, hwaddr addr, case VIRTIO_PCI_COMMON_Q_USEDHI: val = proxy->vqs[vdev->queue_sel].used[1]; break; + case VIRTIO_PCI_COMMON_Q_RESET: + val = proxy->vqs[vdev->queue_sel].reset; + break; default: val = 0; } @@ -1338,6 +1341,7 @@ static void virtio_pci_common_write(void *opaque, hwaddr addr, ((uint64_t)proxy->vqs[vdev->queue_sel].used[1]) << 32 | proxy->vqs[vdev->queue_sel].used[0]); proxy->vqs[vdev->queue_sel].enabled = 1; + proxy->vqs[vdev->queue_sel].reset = 0; } else { virtio_error(vdev, "wrong value for queue_enable %"PRIx64, val); } @@ -1360,6 +1364,16 @@ static void virtio_pci_common_write(void *opaque, hwaddr addr, case VIRTIO_PCI_COMMON_Q_USEDHI: proxy->vqs[vdev->queue_sel].used[1] = val; break; + case VIRTIO_PCI_COMMON_Q_RESET: + if (val == 1) { + proxy->vqs[vdev->queue_sel].reset = 1; + + virtio_queue_reset(vdev, vdev->queue_sel); + + proxy->vqs[vdev->queue_sel].reset = 0; + proxy->vqs[vdev->queue_sel].enabled = 0; + } + break; default: break; } @@ -1954,6 +1968,7 @@ static void virtio_pci_reset(DeviceState *qdev) for (i = 0; i < VIRTIO_QUEUE_MAX; i++) { proxy->vqs[i].enabled = 0; + proxy->vqs[i].reset = 0; proxy->vqs[i].num = 0; proxy->vqs[i].desc[0] = proxy->vqs[i].desc[1] = 0; proxy->vqs[i].avail[0] = proxy->vqs[i].avail[1] = 0; diff --git a/include/hw/virtio/virtio-pci.h b/include/hw/virtio/virtio-pci.h index 2446dcd9ae..938799e8f6 100644 --- a/include/hw/virtio/virtio-pci.h +++ b/include/hw/virtio/virtio-pci.h @@ -117,6 +117,11 @@ typedef struct VirtIOPCIRegion { typedef struct VirtIOPCIQueue { uint16_t num; bool enabled; + /* + * No need to migrate the reset status, because it is always 0 + * when the migration starts. + */ + bool reset; uint32_t desc[2]; uint32_t avail[2]; uint32_t used[2]; -- cgit v1.2.3-55-g7522 From ff48b62809fc3d50e335965c6cb69bec20a495f3 Mon Sep 17 00:00:00 2001 From: Kangjie Xu Date: Mon, 17 Oct 2022 17:25:50 +0800 Subject: vhost: expose vhost_virtqueue_start() Expose vhost_virtqueue_start(), we need to use it when restarting a virtqueue. Signed-off-by: Kangjie Xu Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20221017092558.111082-8-xuanzhuo@linux.alibaba.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- hw/virtio/vhost.c | 8 ++++---- include/hw/virtio/vhost.h | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c index 5185c15295..788d0a0679 100644 --- a/hw/virtio/vhost.c +++ b/hw/virtio/vhost.c @@ -1081,10 +1081,10 @@ out: return ret; } -static int vhost_virtqueue_start(struct vhost_dev *dev, - struct VirtIODevice *vdev, - struct vhost_virtqueue *vq, - unsigned idx) +int vhost_virtqueue_start(struct vhost_dev *dev, + struct VirtIODevice *vdev, + struct vhost_virtqueue *vq, + unsigned idx) { BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev))); VirtioBusState *vbus = VIRTIO_BUS(qbus); diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h index d7eb557885..0054a695dc 100644 --- a/include/hw/virtio/vhost.h +++ b/include/hw/virtio/vhost.h @@ -297,6 +297,9 @@ int vhost_net_set_backend(struct vhost_dev *hdev, int vhost_device_iotlb_miss(struct vhost_dev *dev, uint64_t iova, int write); +int vhost_virtqueue_start(struct vhost_dev *dev, struct VirtIODevice *vdev, + struct vhost_virtqueue *vq, unsigned idx); + void vhost_dev_reset_inflight(struct vhost_inflight *inflight); void vhost_dev_free_inflight(struct vhost_inflight *inflight); void vhost_dev_save_inflight(struct vhost_inflight *inflight, QEMUFile *f); -- cgit v1.2.3-55-g7522 From e1f101d9f60c81b5186098b6dfd196b2730f2070 Mon Sep 17 00:00:00 2001 From: Kangjie Xu Date: Mon, 17 Oct 2022 17:25:51 +0800 Subject: vhost: expose vhost_virtqueue_stop() Expose vhost_virtqueue_stop(), we need to use it when resetting a virtqueue. Signed-off-by: Kangjie Xu Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20221017092558.111082-9-xuanzhuo@linux.alibaba.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- hw/virtio/vhost.c | 8 ++++---- include/hw/virtio/vhost.h | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c index 788d0a0679..d1c4c20b8c 100644 --- a/hw/virtio/vhost.c +++ b/hw/virtio/vhost.c @@ -1201,10 +1201,10 @@ fail_alloc_desc: return r; } -static void vhost_virtqueue_stop(struct vhost_dev *dev, - struct VirtIODevice *vdev, - struct vhost_virtqueue *vq, - unsigned idx) +void vhost_virtqueue_stop(struct vhost_dev *dev, + struct VirtIODevice *vdev, + struct vhost_virtqueue *vq, + unsigned idx) { int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx); struct vhost_vring_state state = { diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h index 0054a695dc..353252ac3e 100644 --- a/include/hw/virtio/vhost.h +++ b/include/hw/virtio/vhost.h @@ -299,6 +299,8 @@ int vhost_device_iotlb_miss(struct vhost_dev *dev, uint64_t iova, int write); int vhost_virtqueue_start(struct vhost_dev *dev, struct VirtIODevice *vdev, struct vhost_virtqueue *vq, unsigned idx); +void vhost_virtqueue_stop(struct vhost_dev *dev, struct VirtIODevice *vdev, + struct vhost_virtqueue *vq, unsigned idx); void vhost_dev_reset_inflight(struct vhost_inflight *inflight); void vhost_dev_free_inflight(struct vhost_inflight *inflight); -- cgit v1.2.3-55-g7522 From c2daa08e1713c4799487bae6daf5d41e024ff736 Mon Sep 17 00:00:00 2001 From: Kangjie Xu Date: Mon, 17 Oct 2022 17:25:52 +0800 Subject: vhost-net: vhost-kernel: introduce vhost_net_virtqueue_reset() Introduce vhost_virtqueue_reset(), which can reset the specific virtqueue in the device. Then it will unmap vrings and the desc of the virtqueue. Here we do not reuse the vhost_net_stop_one() or vhost_dev_stop(), because they work at queue pair level. We do not use vhost_virtqueue_stop() because it may stop the device in the backend. This patch only considers the case of vhost-kernel, when NetClientDriver is NET_CLIENT_DRIVER_TAP. Furthermore, we do not need net->nc->info->poll() because it enables userspace datapath and we want to stop all datapaths for this reset virtqueue here. Signed-off-by: Kangjie Xu Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20221017092558.111082-10-xuanzhuo@linux.alibaba.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- hw/net/vhost_net-stub.c | 6 ++++++ hw/net/vhost_net.c | 25 +++++++++++++++++++++++++ include/net/vhost_net.h | 2 ++ 3 files changed, 33 insertions(+) (limited to 'include') diff --git a/hw/net/vhost_net-stub.c b/hw/net/vhost_net-stub.c index 89d71cfb8e..2d745e359c 100644 --- a/hw/net/vhost_net-stub.c +++ b/hw/net/vhost_net-stub.c @@ -101,3 +101,9 @@ int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu) { return 0; } + +void vhost_net_virtqueue_reset(VirtIODevice *vdev, NetClientState *nc, + int vq_index) +{ + +} diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c index d28f8b974b..8beecb4d22 100644 --- a/hw/net/vhost_net.c +++ b/hw/net/vhost_net.c @@ -531,3 +531,28 @@ int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu) return vhost_ops->vhost_net_set_mtu(&net->dev, mtu); } + +void vhost_net_virtqueue_reset(VirtIODevice *vdev, NetClientState *nc, + int vq_index) +{ + VHostNetState *net = get_vhost_net(nc->peer); + const VhostOps *vhost_ops = net->dev.vhost_ops; + struct vhost_vring_file file = { .fd = -1 }; + int idx; + + /* should only be called after backend is connected */ + assert(vhost_ops); + + idx = vhost_ops->vhost_get_vq_index(&net->dev, vq_index); + + if (net->nc->info->type == NET_CLIENT_DRIVER_TAP) { + file.index = idx; + int r = vhost_net_set_backend(&net->dev, &file); + assert(r >= 0); + } + + vhost_virtqueue_stop(&net->dev, + vdev, + net->dev.vqs + idx, + net->dev.vq_index + idx); +} diff --git a/include/net/vhost_net.h b/include/net/vhost_net.h index 387e913e4e..85d85a4957 100644 --- a/include/net/vhost_net.h +++ b/include/net/vhost_net.h @@ -48,4 +48,6 @@ uint64_t vhost_net_get_acked_features(VHostNetState *net); int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu); +void vhost_net_virtqueue_reset(VirtIODevice *vdev, NetClientState *nc, + int vq_index); #endif -- cgit v1.2.3-55-g7522 From 10f8a115a862045a836932c5d519a848dda5d461 Mon Sep 17 00:00:00 2001 From: Kangjie Xu Date: Mon, 17 Oct 2022 17:25:53 +0800 Subject: vhost-net: vhost-kernel: introduce vhost_net_virtqueue_restart() Introduce vhost_net_virtqueue_restart(), which can restart the specific virtqueue when the vhost net started running before. If it fails to restart the virtqueue, the device will be stopped. Here we do not reuse vhost_net_start_one() or vhost_dev_start() because they work at queue pair level. The mem table and features do not change, so we can call the vhost_virtqueue_start() to restart a specific queue. This patch only considers the case of vhost-kernel, when NetClientDriver is NET_CLIENT_DRIVER_TAP. Signed-off-by: Kangjie Xu Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20221017092558.111082-11-xuanzhuo@linux.alibaba.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- hw/net/vhost_net-stub.c | 6 ++++++ hw/net/vhost_net.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++ include/net/vhost_net.h | 2 ++ 3 files changed, 61 insertions(+) (limited to 'include') diff --git a/hw/net/vhost_net-stub.c b/hw/net/vhost_net-stub.c index 2d745e359c..9f7daae99c 100644 --- a/hw/net/vhost_net-stub.c +++ b/hw/net/vhost_net-stub.c @@ -107,3 +107,9 @@ void vhost_net_virtqueue_reset(VirtIODevice *vdev, NetClientState *nc, { } + +int vhost_net_virtqueue_restart(VirtIODevice *vdev, NetClientState *nc, + int vq_index) +{ + return 0; +} diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c index 8beecb4d22..d2926e2ed6 100644 --- a/hw/net/vhost_net.c +++ b/hw/net/vhost_net.c @@ -34,6 +34,7 @@ #include "standard-headers/linux/virtio_ring.h" #include "hw/virtio/vhost.h" #include "hw/virtio/virtio-bus.h" +#include "linux-headers/linux/vhost.h" /* Features supported by host kernel. */ @@ -556,3 +557,55 @@ void vhost_net_virtqueue_reset(VirtIODevice *vdev, NetClientState *nc, net->dev.vqs + idx, net->dev.vq_index + idx); } + +int vhost_net_virtqueue_restart(VirtIODevice *vdev, NetClientState *nc, + int vq_index) +{ + VHostNetState *net = get_vhost_net(nc->peer); + const VhostOps *vhost_ops = net->dev.vhost_ops; + struct vhost_vring_file file = { }; + int idx, r; + + if (!net->dev.started) { + return -EBUSY; + } + + /* should only be called after backend is connected */ + assert(vhost_ops); + + idx = vhost_ops->vhost_get_vq_index(&net->dev, vq_index); + + r = vhost_virtqueue_start(&net->dev, + vdev, + net->dev.vqs + idx, + net->dev.vq_index + idx); + if (r < 0) { + goto err_start; + } + + if (net->nc->info->type == NET_CLIENT_DRIVER_TAP) { + file.index = idx; + file.fd = net->backend; + r = vhost_net_set_backend(&net->dev, &file); + if (r < 0) { + r = -errno; + goto err_start; + } + } + + return 0; + +err_start: + error_report("Error when restarting the queue."); + + if (net->nc->info->type == NET_CLIENT_DRIVER_TAP) { + file.fd = VHOST_FILE_UNBIND; + file.index = idx; + int r = vhost_net_set_backend(&net->dev, &file); + assert(r >= 0); + } + + vhost_dev_stop(&net->dev, vdev); + + return r; +} diff --git a/include/net/vhost_net.h b/include/net/vhost_net.h index 85d85a4957..40b9a40074 100644 --- a/include/net/vhost_net.h +++ b/include/net/vhost_net.h @@ -50,4 +50,6 @@ int vhost_net_set_mtu(struct vhost_net *net, uint16_t mtu); void vhost_net_virtqueue_reset(VirtIODevice *vdev, NetClientState *nc, int vq_index); +int vhost_net_virtqueue_restart(VirtIODevice *vdev, NetClientState *nc, + int vq_index); #endif -- cgit v1.2.3-55-g7522 From 5dbad0af10ff7d4f83db7f4ddd872d85332ebecc Mon Sep 17 00:00:00 2001 From: Igor Mammedov Date: Mon, 17 Oct 2022 12:21:41 +0200 Subject: acpi: add get_dev_aml_func() helper It will be used in followup commits to figure out if device has it's own, device specific AML block. Signed-off-by: Igor Mammedov Message-Id: <20221017102146.2254096-7-imammedo@redhat.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin Reviewed-by: Ani Sinha --- include/hw/acpi/acpi_aml_interface.h | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/hw/acpi/acpi_aml_interface.h b/include/hw/acpi/acpi_aml_interface.h index ab76f0e55d..436da069d6 100644 --- a/include/hw/acpi/acpi_aml_interface.h +++ b/include/hw/acpi/acpi_aml_interface.h @@ -29,11 +29,20 @@ struct AcpiDevAmlIfClass { dev_aml_fn build_dev_aml; }; -static inline void call_dev_aml_func(DeviceState *dev, Aml *scope) +static inline dev_aml_fn get_dev_aml_func(DeviceState *dev) { if (object_dynamic_cast(OBJECT(dev), TYPE_ACPI_DEV_AML_IF)) { AcpiDevAmlIfClass *klass = ACPI_DEV_AML_IF_GET_CLASS(dev); - klass->build_dev_aml(ACPI_DEV_AML_IF(dev), scope); + return klass->build_dev_aml; + } + return NULL; +} + +static inline void call_dev_aml_func(DeviceState *dev, Aml *scope) +{ + dev_aml_fn fn = get_dev_aml_func(dev); + if (fn) { + fn(ACPI_DEV_AML_IF(dev), scope); } } -- cgit v1.2.3-55-g7522 From 15377f6e79cc6aa08dbafe82607e0bda13ca44b5 Mon Sep 17 00:00:00 2001 From: Akihiko Odaki Date: Mon, 29 Aug 2022 17:35:24 +0900 Subject: msix: Assert that specified vector is in range There were several different ways to deal with the situation where the vector specified for a msix function is out of bound: - early return a function and keep progresssing - propagate the error to the caller - mark msix unusable - assert it is in bound - just ignore An out-of-bound vector should not be specified if the device implementation is correct so let msix functions always assert that the specified vector is in range. An exceptional case is virtio-pci, which allows the guest to configure vectors. For virtio-pci, it is more appropriate to introduce its own checks because it is sometimes too late to check the vector range in msix functions. Signed-off-by: Akihiko Odaki Message-Id: <20220829083524.143640-1-akihiko.odaki@daynix.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin Reviewed-by: Yuval Shaia Signed-off-by: Akihiko Odaki <akihiko.odaki@daynix.com>
--- hw/net/e1000e.c | 15 +++-------- hw/net/rocker/rocker.c | 23 +++------------- hw/net/vmxnet3.c | 27 ++++--------------- hw/nvme/ctrl.c | 5 +--- hw/pci/msix.c | 24 +++++++---------- hw/rdma/vmw/pvrdma_main.c | 7 +---- hw/remote/vfio-user-obj.c | 9 +------ hw/virtio/virtio-pci.c | 67 ++++++++++++++++++++++++++++++++++------------- include/hw/pci/msix.h | 4 +-- 9 files changed, 74 insertions(+), 107 deletions(-) (limited to 'include') diff --git a/hw/net/e1000e.c b/hw/net/e1000e.c index ac96f7665a..7523e9f5d2 100644 --- a/hw/net/e1000e.c +++ b/hw/net/e1000e.c @@ -276,25 +276,18 @@ e1000e_unuse_msix_vectors(E1000EState *s, int num_vectors) } } -static bool +static void e1000e_use_msix_vectors(E1000EState *s, int num_vectors) { int i; for (i = 0; i < num_vectors; i++) { - int res = msix_vector_use(PCI_DEVICE(s), i); - if (res < 0) { - trace_e1000e_msix_use_vector_fail(i, res); - e1000e_unuse_msix_vectors(s, i); - return false; - } + msix_vector_use(PCI_DEVICE(s), i); } - return true; } static void e1000e_init_msix(E1000EState *s) { - PCIDevice *d = PCI_DEVICE(s); int res = msix_init(PCI_DEVICE(s), E1000E_MSIX_VEC_NUM, &s->msix, E1000E_MSIX_IDX, E1000E_MSIX_TABLE, @@ -305,9 +298,7 @@ e1000e_init_msix(E1000EState *s) if (res < 0) { trace_e1000e_msix_init_fail(res); } else { - if (!e1000e_use_msix_vectors(s, E1000E_MSIX_VEC_NUM)) { - msix_uninit(d, &s->msix, &s->msix); - } + e1000e_use_msix_vectors(s, E1000E_MSIX_VEC_NUM); } } diff --git a/hw/net/rocker/rocker.c b/hw/net/rocker/rocker.c index d8f3f16fe8..281d43e6cf 100644 --- a/hw/net/rocker/rocker.c +++ b/hw/net/rocker/rocker.c @@ -1212,24 +1212,14 @@ static void rocker_msix_vectors_unuse(Rocker *r, } } -static int rocker_msix_vectors_use(Rocker *r, - unsigned int num_vectors) +static void rocker_msix_vectors_use(Rocker *r, unsigned int num_vectors) { PCIDevice *dev = PCI_DEVICE(r); - int err; int i; for (i = 0; i < num_vectors; i++) { - err = msix_vector_use(dev, i); - if (err) { - goto rollback; - } + msix_vector_use(dev, i); } - return 0; - -rollback: - rocker_msix_vectors_unuse(r, i); - return err; } static int rocker_msix_init(Rocker *r, Error **errp) @@ -1247,16 +1237,9 @@ static int rocker_msix_init(Rocker *r, Error **errp) return err; } - err = rocker_msix_vectors_use(r, ROCKER_MSIX_VEC_COUNT(r->fp_ports)); - if (err) { - goto err_msix_vectors_use; - } + rocker_msix_vectors_use(r, ROCKER_MSIX_VEC_COUNT(r->fp_ports)); return 0; - -err_msix_vectors_use: - msix_uninit(dev, &r->msix_bar, &r->msix_bar); - return err; } static void rocker_msix_uninit(Rocker *r) diff --git a/hw/net/vmxnet3.c b/hw/net/vmxnet3.c index 0b7acf7f89..d2ab527ef4 100644 --- a/hw/net/vmxnet3.c +++ b/hw/net/vmxnet3.c @@ -2110,20 +2110,14 @@ vmxnet3_unuse_msix_vectors(VMXNET3State *s, int num_vectors) } } -static bool +static void vmxnet3_use_msix_vectors(VMXNET3State *s, int num_vectors) { PCIDevice *d = PCI_DEVICE(s); int i; for (i = 0; i < num_vectors; i++) { - int res = msix_vector_use(d, i); - if (0 > res) { - VMW_WRPRN("Failed to use MSI-X vector %d, error %d", i, res); - vmxnet3_unuse_msix_vectors(s, i); - return false; - } + msix_vector_use(d, i); } - return true; } static bool @@ -2141,13 +2135,8 @@ vmxnet3_init_msix(VMXNET3State *s) VMW_WRPRN("Failed to initialize MSI-X, error %d", res); s->msix_used = false; } else { - if (!vmxnet3_use_msix_vectors(s, VMXNET3_MAX_INTRS)) { - VMW_WRPRN("Failed to use MSI-X vectors, error %d", res); - msix_uninit(d, &s->msix_bar, &s->msix_bar); - s->msix_used = false; - } else { - s->msix_used = true; - } + vmxnet3_use_msix_vectors(s, VMXNET3_MAX_INTRS); + s->msix_used = true; } return s->msix_used; } @@ -2412,19 +2401,13 @@ static const VMStateDescription vmstate_vmxnet3_rxq_descr = { static int vmxnet3_post_load(void *opaque, int version_id) { VMXNET3State *s = opaque; - PCIDevice *d = PCI_DEVICE(s); net_tx_pkt_init(&s->tx_pkt, PCI_DEVICE(s), s->max_tx_frags, s->peer_has_vhdr); net_rx_pkt_init(&s->rx_pkt, s->peer_has_vhdr); if (s->msix_used) { - if (!vmxnet3_use_msix_vectors(s, VMXNET3_MAX_INTRS)) { - VMW_WRPRN("Failed to re-use MSI-X vectors"); - msix_uninit(d, &s->msix_bar, &s->msix_bar); - s->msix_used = false; - return -1; - } + vmxnet3_use_msix_vectors(s, VMXNET3_MAX_INTRS); } if (!vmxnet3_validate_queues(s)) { diff --git a/hw/nvme/ctrl.c b/hw/nvme/ctrl.c index 87aeba0564..d38fdd990e 100644 --- a/hw/nvme/ctrl.c +++ b/hw/nvme/ctrl.c @@ -4744,11 +4744,8 @@ static void nvme_init_cq(NvmeCQueue *cq, NvmeCtrl *n, uint64_t dma_addr, uint16_t cqid, uint16_t vector, uint16_t size, uint16_t irq_enabled) { - int ret; - if (msix_enabled(&n->parent_obj)) { - ret = msix_vector_use(&n->parent_obj, vector); - assert(ret == 0); + msix_vector_use(&n->parent_obj, vector); } cq->ctrl = n; cq->cqid = cqid; diff --git a/hw/pci/msix.c b/hw/pci/msix.c index 1e381a9813..9e70fcd6fa 100644 --- a/hw/pci/msix.c +++ b/hw/pci/msix.c @@ -136,17 +136,12 @@ static void msix_handle_mask_update(PCIDevice *dev, int vector, bool was_masked) } } -void msix_set_mask(PCIDevice *dev, int vector, bool mask, Error **errp) +void msix_set_mask(PCIDevice *dev, int vector, bool mask) { - ERRP_GUARD(); unsigned offset; bool was_masked; - if (vector > dev->msix_entries_nr) { - error_setg(errp, "msix: vector %d not allocated. max vector is %d", - vector, dev->msix_entries_nr); - return; - } + assert(vector < dev->msix_entries_nr); offset = vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL; @@ -522,7 +517,9 @@ void msix_notify(PCIDevice *dev, unsigned vector) { MSIMessage msg; - if (vector >= dev->msix_entries_nr || !dev->msix_entry_used[vector]) { + assert(vector < dev->msix_entries_nr); + + if (!dev->msix_entry_used[vector]) { return; } @@ -558,20 +555,17 @@ void msix_reset(PCIDevice *dev) * don't want to follow the spec suggestion can declare all vectors as used. */ /* Mark vector as used. */ -int msix_vector_use(PCIDevice *dev, unsigned vector) +void msix_vector_use(PCIDevice *dev, unsigned vector) { - if (vector >= dev->msix_entries_nr) { - return -EINVAL; - } - + assert(vector < dev->msix_entries_nr); dev->msix_entry_used[vector]++; - return 0; } /* Mark vector as unused. */ void msix_vector_unuse(PCIDevice *dev, unsigned vector) { - if (vector >= dev->msix_entries_nr || !dev->msix_entry_used[vector]) { + assert(vector < dev->msix_entries_nr); + if (!dev->msix_entry_used[vector]) { return; } if (--dev->msix_entry_used[vector]) { diff --git a/hw/rdma/vmw/pvrdma_main.c b/hw/rdma/vmw/pvrdma_main.c index 58db0b8e3b..4fc6712025 100644 --- a/hw/rdma/vmw/pvrdma_main.c +++ b/hw/rdma/vmw/pvrdma_main.c @@ -307,12 +307,7 @@ static int init_msix(PCIDevice *pdev) } for (i = 0; i < RDMA_MAX_INTRS; i++) { - rc = msix_vector_use(PCI_DEVICE(dev), i); - if (rc < 0) { - rdma_error_report("Fail mark MSI-X vector %d", i); - uninit_msix(pdev, i); - return rc; - } + msix_vector_use(PCI_DEVICE(dev), i); } return 0; diff --git a/hw/remote/vfio-user-obj.c b/hw/remote/vfio-user-obj.c index c6cc53acf2..4e36bb8bcf 100644 --- a/hw/remote/vfio-user-obj.c +++ b/hw/remote/vfio-user-obj.c @@ -602,17 +602,10 @@ static void vfu_msix_irq_state(vfu_ctx_t *vfu_ctx, uint32_t start, uint32_t count, bool mask) { VfuObject *o = vfu_get_private(vfu_ctx); - Error *err = NULL; uint32_t vector; for (vector = start; vector < count; vector++) { - msix_set_mask(o->pci_dev, vector, mask, &err); - if (err) { - VFU_OBJECT_ERROR(o, "vfu: %s: %s", o->device, - error_get_pretty(err)); - error_free(err); - err = NULL; - } + msix_set_mask(o->pci_dev, vector, mask); } } diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c index 855718d586..a1c9dfa7bb 100644 --- a/hw/virtio/virtio-pci.c +++ b/hw/virtio/virtio-pci.c @@ -71,9 +71,11 @@ static void virtio_pci_notify(DeviceState *d, uint16_t vector) { VirtIOPCIProxy *proxy = to_virtio_pci_proxy_fast(d); - if (msix_enabled(&proxy->pci_dev)) - msix_notify(&proxy->pci_dev, vector); - else { + if (msix_enabled(&proxy->pci_dev)) { + if (vector != VIRTIO_NO_VECTOR) { + msix_notify(&proxy->pci_dev, vector); + } + } else { VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); pci_set_irq(&proxy->pci_dev, qatomic_read(&vdev->isr) & 1); } @@ -175,6 +177,7 @@ static int virtio_pci_load_config(DeviceState *d, QEMUFile *f) { VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d); VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); + uint16_t vector; int ret; ret = pci_device_load(&proxy->pci_dev, f); @@ -184,12 +187,17 @@ static int virtio_pci_load_config(DeviceState *d, QEMUFile *f) msix_unuse_all_vectors(&proxy->pci_dev); msix_load(&proxy->pci_dev, f); if (msix_present(&proxy->pci_dev)) { - qemu_get_be16s(f, &vdev->config_vector); + qemu_get_be16s(f, &vector); + + if (vector != VIRTIO_NO_VECTOR && vector >= proxy->nvectors) { + return -EINVAL; + } } else { - vdev->config_vector = VIRTIO_NO_VECTOR; + vector = VIRTIO_NO_VECTOR; } - if (vdev->config_vector != VIRTIO_NO_VECTOR) { - return msix_vector_use(&proxy->pci_dev, vdev->config_vector); + vdev->config_vector = vector; + if (vector != VIRTIO_NO_VECTOR) { + msix_vector_use(&proxy->pci_dev, vector); } return 0; } @@ -202,12 +210,15 @@ static int virtio_pci_load_queue(DeviceState *d, int n, QEMUFile *f) uint16_t vector; if (msix_present(&proxy->pci_dev)) { qemu_get_be16s(f, &vector); + if (vector != VIRTIO_NO_VECTOR && vector >= proxy->nvectors) { + return -EINVAL; + } } else { vector = VIRTIO_NO_VECTOR; } virtio_queue_set_vector(vdev, n, vector); if (vector != VIRTIO_NO_VECTOR) { - return msix_vector_use(&proxy->pci_dev, vector); + msix_vector_use(&proxy->pci_dev, vector); } return 0; @@ -299,6 +310,7 @@ static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val) { VirtIOPCIProxy *proxy = opaque; VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); + uint16_t vector; hwaddr pa; switch (addr) { @@ -352,18 +364,28 @@ static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val) } break; case VIRTIO_MSI_CONFIG_VECTOR: - msix_vector_unuse(&proxy->pci_dev, vdev->config_vector); + if (vdev->config_vector != VIRTIO_NO_VECTOR) { + msix_vector_unuse(&proxy->pci_dev, vdev->config_vector); + } /* Make it possible for guest to discover an error took place. */ - if (msix_vector_use(&proxy->pci_dev, val) < 0) + if (val < proxy->nvectors) { + msix_vector_use(&proxy->pci_dev, val); + } else { val = VIRTIO_NO_VECTOR; + } vdev->config_vector = val; break; case VIRTIO_MSI_QUEUE_VECTOR: - msix_vector_unuse(&proxy->pci_dev, - virtio_queue_vector(vdev, vdev->queue_sel)); + vector = virtio_queue_vector(vdev, vdev->queue_sel); + if (vector != VIRTIO_NO_VECTOR) { + msix_vector_unuse(&proxy->pci_dev, vector); + } /* Make it possible for guest to discover an error took place. */ - if (msix_vector_use(&proxy->pci_dev, val) < 0) + if (val < proxy->nvectors) { + msix_vector_use(&proxy->pci_dev, val); + } else { val = VIRTIO_NO_VECTOR; + } virtio_queue_set_vector(vdev, vdev->queue_sel, val); break; default: @@ -1266,6 +1288,7 @@ static void virtio_pci_common_write(void *opaque, hwaddr addr, { VirtIOPCIProxy *proxy = opaque; VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus); + uint16_t vector; if (vdev == NULL) { return; @@ -1287,9 +1310,13 @@ static void virtio_pci_common_write(void *opaque, hwaddr addr, } break; case VIRTIO_PCI_COMMON_MSIX: - msix_vector_unuse(&proxy->pci_dev, vdev->config_vector); + if (vdev->config_vector != VIRTIO_NO_VECTOR) { + msix_vector_unuse(&proxy->pci_dev, vdev->config_vector); + } /* Make it possible for guest to discover an error took place. */ - if (msix_vector_use(&proxy->pci_dev, val) < 0) { + if (val < proxy->nvectors) { + msix_vector_use(&proxy->pci_dev, val); + } else { val = VIRTIO_NO_VECTOR; } vdev->config_vector = val; @@ -1321,10 +1348,14 @@ static void virtio_pci_common_write(void *opaque, hwaddr addr, proxy->vqs[vdev->queue_sel].num); break; case VIRTIO_PCI_COMMON_Q_MSIX: - msix_vector_unuse(&proxy->pci_dev, - virtio_queue_vector(vdev, vdev->queue_sel)); + vector = virtio_queue_vector(vdev, vdev->queue_sel); + if (vector != VIRTIO_NO_VECTOR) { + msix_vector_unuse(&proxy->pci_dev, vector); + } /* Make it possible for guest to discover an error took place. */ - if (msix_vector_use(&proxy->pci_dev, val) < 0) { + if (val < proxy->nvectors) { + msix_vector_use(&proxy->pci_dev, val); + } else { val = VIRTIO_NO_VECTOR; } virtio_queue_set_vector(vdev, vdev->queue_sel, val); diff --git a/include/hw/pci/msix.h b/include/hw/pci/msix.h index 4f1cda0ebe..0e6f257e45 100644 --- a/include/hw/pci/msix.h +++ b/include/hw/pci/msix.h @@ -33,10 +33,10 @@ bool msix_is_masked(PCIDevice *dev, unsigned vector); void msix_set_pending(PCIDevice *dev, unsigned vector); void msix_clr_pending(PCIDevice *dev, int vector); -int msix_vector_use(PCIDevice *dev, unsigned vector); +void msix_vector_use(PCIDevice *dev, unsigned vector); void msix_vector_unuse(PCIDevice *dev, unsigned vector); void msix_unuse_all_vectors(PCIDevice *dev); -void msix_set_mask(PCIDevice *dev, int vector, bool mask, Error **errp); +void msix_set_mask(PCIDevice *dev, int vector, bool mask); void msix_notify(PCIDevice *dev, unsigned vector); -- cgit v1.2.3-55-g7522 From baa44bce87fe53ef5c95d39e634b3bace014d235 Mon Sep 17 00:00:00 2001 From: Cindy Lu Date: Mon, 31 Oct 2022 11:10:19 +0800 Subject: vfio: move implement of vfio_get_xlat_addr() to memory.c - Move the implement vfio_get_xlat_addr to softmmu/memory.c, and change the name to memory_get_xlat_addr(). So we can use this function on other devices, such as vDPA device. - Add a new function vfio_get_xlat_addr in vfio/common.c, and it will check whether the memory is backed by a discard manager. then device can have its own warning. Signed-off-by: Cindy Lu Message-Id: <20221031031020.1405111-2-lulu@redhat.com> Acked-by: Alex Williamson Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- hw/vfio/common.c | 66 ++++------------------------------------------ include/exec/memory.h | 4 +++ softmmu/memory.c | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 61 deletions(-) (limited to 'include') diff --git a/hw/vfio/common.c b/hw/vfio/common.c index 6b5d8c0bf6..130e5d1dc7 100644 --- a/hw/vfio/common.c +++ b/hw/vfio/common.c @@ -578,45 +578,11 @@ static bool vfio_listener_skipped_section(MemoryRegionSection *section) static bool vfio_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr, ram_addr_t *ram_addr, bool *read_only) { - MemoryRegion *mr; - hwaddr xlat; - hwaddr len = iotlb->addr_mask + 1; - bool writable = iotlb->perm & IOMMU_WO; - - /* - * The IOMMU TLB entry we have just covers translation through - * this IOMMU to its immediate target. We need to translate - * it the rest of the way through to memory. - */ - mr = address_space_translate(&address_space_memory, - iotlb->translated_addr, - &xlat, &len, writable, - MEMTXATTRS_UNSPECIFIED); - if (!memory_region_is_ram(mr)) { - error_report("iommu map to non memory area %"HWADDR_PRIx"", - xlat); - return false; - } else if (memory_region_has_ram_discard_manager(mr)) { - RamDiscardManager *rdm = memory_region_get_ram_discard_manager(mr); - MemoryRegionSection tmp = { - .mr = mr, - .offset_within_region = xlat, - .size = int128_make64(len), - }; - - /* - * Malicious VMs can map memory into the IOMMU, which is expected - * to remain discarded. vfio will pin all pages, populating memory. - * Disallow that. vmstate priorities make sure any RamDiscardManager - * were already restored before IOMMUs are restored. - */ - if (!ram_discard_manager_is_populated(rdm, &tmp)) { - error_report("iommu map to discarded memory (e.g., unplugged via" - " virtio-mem): %"HWADDR_PRIx"", - iotlb->translated_addr); - return false; - } + bool ret, mr_has_discard_manager; + ret = memory_get_xlat_addr(iotlb, vaddr, ram_addr, read_only, + &mr_has_discard_manager); + if (ret && mr_has_discard_manager) { /* * Malicious VMs might trigger discarding of IOMMU-mapped memory. The * pages will remain pinned inside vfio until unmapped, resulting in a @@ -635,29 +601,7 @@ static bool vfio_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr, " intended via an IOMMU. It's possible to mitigate " " by setting/adjusting RLIMIT_MEMLOCK."); } - - /* - * Translation truncates length to the IOMMU page size, - * check that it did not truncate too much. - */ - if (len & iotlb->addr_mask) { - error_report("iommu has granularity incompatible with target AS"); - return false; - } - - if (vaddr) { - *vaddr = memory_region_get_ram_ptr(mr) + xlat; - } - - if (ram_addr) { - *ram_addr = memory_region_get_ram_addr(mr) + xlat; - } - - if (read_only) { - *read_only = !writable || mr->readonly; - } - - return true; + return ret; } static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb) diff --git a/include/exec/memory.h b/include/exec/memory.h index bfb1de8eea..d1e79c39dc 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -713,6 +713,10 @@ void ram_discard_manager_register_listener(RamDiscardManager *rdm, void ram_discard_manager_unregister_listener(RamDiscardManager *rdm, RamDiscardListener *rdl); +bool memory_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr, + ram_addr_t *ram_addr, bool *read_only, + bool *mr_has_discard_manager); + typedef struct CoalescedMemoryRange CoalescedMemoryRange; typedef struct MemoryRegionIoeventfd MemoryRegionIoeventfd; diff --git a/softmmu/memory.c b/softmmu/memory.c index 7ba2048836..bc0be3f62c 100644 --- a/softmmu/memory.c +++ b/softmmu/memory.c @@ -33,6 +33,7 @@ #include "qemu/accel.h" #include "hw/boards.h" #include "migration/vmstate.h" +#include "exec/address-spaces.h" //#define DEBUG_UNASSIGNED @@ -2121,6 +2122,77 @@ void ram_discard_manager_unregister_listener(RamDiscardManager *rdm, rdmc->unregister_listener(rdm, rdl); } +/* Called with rcu_read_lock held. */ +bool memory_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr, + ram_addr_t *ram_addr, bool *read_only, + bool *mr_has_discard_manager) +{ + MemoryRegion *mr; + hwaddr xlat; + hwaddr len = iotlb->addr_mask + 1; + bool writable = iotlb->perm & IOMMU_WO; + + if (mr_has_discard_manager) { + *mr_has_discard_manager = false; + } + /* + * The IOMMU TLB entry we have just covers translation through + * this IOMMU to its immediate target. We need to translate + * it the rest of the way through to memory. + */ + mr = address_space_translate(&address_space_memory, iotlb->translated_addr, + &xlat, &len, writable, MEMTXATTRS_UNSPECIFIED); + if (!memory_region_is_ram(mr)) { + error_report("iommu map to non memory area %" HWADDR_PRIx "", xlat); + return false; + } else if (memory_region_has_ram_discard_manager(mr)) { + RamDiscardManager *rdm = memory_region_get_ram_discard_manager(mr); + MemoryRegionSection tmp = { + .mr = mr, + .offset_within_region = xlat, + .size = int128_make64(len), + }; + if (mr_has_discard_manager) { + *mr_has_discard_manager = true; + } + /* + * Malicious VMs can map memory into the IOMMU, which is expected + * to remain discarded. vfio will pin all pages, populating memory. + * Disallow that. vmstate priorities make sure any RamDiscardManager + * were already restored before IOMMUs are restored. + */ + if (!ram_discard_manager_is_populated(rdm, &tmp)) { + error_report("iommu map to discarded memory (e.g., unplugged via" + " virtio-mem): %" HWADDR_PRIx "", + iotlb->translated_addr); + return false; + } + } + + /* + * Translation truncates length to the IOMMU page size, + * check that it did not truncate too much. + */ + if (len & iotlb->addr_mask) { + error_report("iommu has granularity incompatible with target AS"); + return false; + } + + if (vaddr) { + *vaddr = memory_region_get_ram_ptr(mr) + xlat; + } + + if (ram_addr) { + *ram_addr = memory_region_get_ram_addr(mr) + xlat; + } + + if (read_only) { + *read_only = !writable || mr->readonly; + } + + return true; +} + void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client) { uint8_t mask = 1 << client; -- cgit v1.2.3-55-g7522 From da8d439c8048f685e0333ae468b7520b82925e75 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 28 Oct 2022 14:14:34 +0800 Subject: intel-iommu: drop VTDBus We introduce VTDBus structure as an intermediate step for searching the address space. This works well with SID based matching/lookup. But when we want to support SID plus PASID based address space lookup, this intermediate steps turns out to be a burden. So the patch simply drops the VTDBus structure and use the PCIBus and devfn as the key for the g_hash_table(). This simplifies the codes and the future PASID extension. To prevent being slower for past vtd_find_as_from_bus_num() callers, a vtd_as cache indexed by the bus number is introduced to store the last recent search result of a vtd_as belongs to a specific bus. Reviewed-by: Peter Xu Signed-off-by: Jason Wang Message-Id: <20221028061436.30093-3-jasowang@redhat.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin Reviewed-by: Yi Liu --- hw/i386/intel_iommu.c | 234 +++++++++++++++++++++--------------------- include/hw/i386/intel_iommu.h | 11 +- 2 files changed, 118 insertions(+), 127 deletions(-) (limited to 'include') diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c index 271de995be..3d426bb326 100644 --- a/hw/i386/intel_iommu.c +++ b/hw/i386/intel_iommu.c @@ -61,6 +61,16 @@ } \ } +/* + * PCI bus number (or SID) is not reliable since the device is usaully + * initalized before guest can configure the PCI bridge + * (SECONDARY_BUS_NUMBER). + */ +struct vtd_as_key { + PCIBus *bus; + uint8_t devfn; +}; + static void vtd_address_space_refresh_all(IntelIOMMUState *s); static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n); @@ -210,6 +220,27 @@ static guint vtd_uint64_hash(gconstpointer v) return (guint)*(const uint64_t *)v; } +static gboolean vtd_as_equal(gconstpointer v1, gconstpointer v2) +{ + const struct vtd_as_key *key1 = v1; + const struct vtd_as_key *key2 = v2; + + return (key1->bus == key2->bus) && (key1->devfn == key2->devfn); +} + +/* + * Note that we use pointer to PCIBus as the key, so hashing/shifting + * based on the pointer value is intended. Note that we deal with + * collisions through vtd_as_equal(). + */ +static guint vtd_as_hash(gconstpointer v) +{ + const struct vtd_as_key *key = v; + guint value = (guint)(uintptr_t)key->bus; + + return (guint)(value << 8 | key->devfn); +} + static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value, gpointer user_data) { @@ -248,22 +279,14 @@ static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value, static void vtd_reset_context_cache_locked(IntelIOMMUState *s) { VTDAddressSpace *vtd_as; - VTDBus *vtd_bus; - GHashTableIter bus_it; - uint32_t devfn_it; + GHashTableIter as_it; trace_vtd_context_cache_reset(); - g_hash_table_iter_init(&bus_it, s->vtd_as_by_busptr); + g_hash_table_iter_init(&as_it, s->vtd_address_spaces); - while (g_hash_table_iter_next (&bus_it, NULL, (void**)&vtd_bus)) { - for (devfn_it = 0; devfn_it < PCI_DEVFN_MAX; ++devfn_it) { - vtd_as = vtd_bus->dev_as[devfn_it]; - if (!vtd_as) { - continue; - } - vtd_as->context_cache_entry.context_cache_gen = 0; - } + while (g_hash_table_iter_next(&as_it, NULL, (void **)&vtd_as)) { + vtd_as->context_cache_entry.context_cache_gen = 0; } s->context_cache_gen = 1; } @@ -993,32 +1016,6 @@ static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level) return slpte & rsvd_mask; } -/* Find the VTD address space associated with a given bus number */ -static VTDBus *vtd_find_as_from_bus_num(IntelIOMMUState *s, uint8_t bus_num) -{ - VTDBus *vtd_bus = s->vtd_as_by_bus_num[bus_num]; - GHashTableIter iter; - - if (vtd_bus) { - return vtd_bus; - } - - /* - * Iterate over the registered buses to find the one which - * currently holds this bus number and update the bus_num - * lookup table. - */ - g_hash_table_iter_init(&iter, s->vtd_as_by_busptr); - while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_bus)) { - if (pci_bus_num(vtd_bus->bus) == bus_num) { - s->vtd_as_by_bus_num[bus_num] = vtd_bus; - return vtd_bus; - } - } - - return NULL; -} - /* Given the @iova, get relevant @slptep. @slpte_level will be the last level * of the translation, can be used for deciding the size of large page. */ @@ -1632,24 +1629,13 @@ static bool vtd_switch_address_space(VTDAddressSpace *as) static void vtd_switch_address_space_all(IntelIOMMUState *s) { + VTDAddressSpace *vtd_as; GHashTableIter iter; - VTDBus *vtd_bus; - int i; - - g_hash_table_iter_init(&iter, s->vtd_as_by_busptr); - while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_bus)) { - for (i = 0; i < PCI_DEVFN_MAX; i++) { - if (!vtd_bus->dev_as[i]) { - continue; - } - vtd_switch_address_space(vtd_bus->dev_as[i]); - } - } -} -static inline uint16_t vtd_make_source_id(uint8_t bus_num, uint8_t devfn) -{ - return ((bus_num & 0xffUL) << 8) | (devfn & 0xffUL); + g_hash_table_iter_init(&iter, s->vtd_address_spaces); + while (g_hash_table_iter_next(&iter, NULL, (void **)&vtd_as)) { + vtd_switch_address_space(vtd_as); + } } static const bool vtd_qualified_faults[] = { @@ -1686,18 +1672,37 @@ static inline bool vtd_is_interrupt_addr(hwaddr addr) return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST; } +static gboolean vtd_find_as_by_sid(gpointer key, gpointer value, + gpointer user_data) +{ + struct vtd_as_key *as_key = (struct vtd_as_key *)key; + uint16_t target_sid = *(uint16_t *)user_data; + uint16_t sid = PCI_BUILD_BDF(pci_bus_num(as_key->bus), as_key->devfn); + return sid == target_sid; +} + +static VTDAddressSpace *vtd_get_as_by_sid(IntelIOMMUState *s, uint16_t sid) +{ + uint8_t bus_num = PCI_BUS_NUM(sid); + VTDAddressSpace *vtd_as = s->vtd_as_cache[bus_num]; + + if (vtd_as && + (sid == PCI_BUILD_BDF(pci_bus_num(vtd_as->bus), vtd_as->devfn))) { + return vtd_as; + } + + vtd_as = g_hash_table_find(s->vtd_address_spaces, vtd_find_as_by_sid, &sid); + s->vtd_as_cache[bus_num] = vtd_as; + + return vtd_as; +} + static void vtd_pt_enable_fast_path(IntelIOMMUState *s, uint16_t source_id) { - VTDBus *vtd_bus; VTDAddressSpace *vtd_as; bool success = false; - vtd_bus = vtd_find_as_from_bus_num(s, VTD_SID_TO_BUS(source_id)); - if (!vtd_bus) { - goto out; - } - - vtd_as = vtd_bus->dev_as[VTD_SID_TO_DEVFN(source_id)]; + vtd_as = vtd_get_as_by_sid(s, source_id); if (!vtd_as) { goto out; } @@ -1733,7 +1738,7 @@ static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, VTDContextCacheEntry *cc_entry; uint64_t slpte, page_mask; uint32_t level; - uint16_t source_id = vtd_make_source_id(bus_num, devfn); + uint16_t source_id = PCI_BUILD_BDF(bus_num, devfn); int ret_fr; bool is_fpd_set = false; bool reads = true; @@ -1905,11 +1910,10 @@ static void vtd_context_device_invalidate(IntelIOMMUState *s, uint16_t source_id, uint16_t func_mask) { + GHashTableIter as_it; uint16_t mask; - VTDBus *vtd_bus; VTDAddressSpace *vtd_as; uint8_t bus_n, devfn; - uint16_t devfn_it; trace_vtd_inv_desc_cc_devices(source_id, func_mask); @@ -1932,32 +1936,31 @@ static void vtd_context_device_invalidate(IntelIOMMUState *s, mask = ~mask; bus_n = VTD_SID_TO_BUS(source_id); - vtd_bus = vtd_find_as_from_bus_num(s, bus_n); - if (vtd_bus) { - devfn = VTD_SID_TO_DEVFN(source_id); - for (devfn_it = 0; devfn_it < PCI_DEVFN_MAX; ++devfn_it) { - vtd_as = vtd_bus->dev_as[devfn_it]; - if (vtd_as && ((devfn_it & mask) == (devfn & mask))) { - trace_vtd_inv_desc_cc_device(bus_n, VTD_PCI_SLOT(devfn_it), - VTD_PCI_FUNC(devfn_it)); - vtd_iommu_lock(s); - vtd_as->context_cache_entry.context_cache_gen = 0; - vtd_iommu_unlock(s); - /* - * Do switch address space when needed, in case if the - * device passthrough bit is switched. - */ - vtd_switch_address_space(vtd_as); - /* - * So a device is moving out of (or moving into) a - * domain, resync the shadow page table. - * This won't bring bad even if we have no such - * notifier registered - the IOMMU notification - * framework will skip MAP notifications if that - * happened. - */ - vtd_sync_shadow_page_table(vtd_as); - } + devfn = VTD_SID_TO_DEVFN(source_id); + + g_hash_table_iter_init(&as_it, s->vtd_address_spaces); + while (g_hash_table_iter_next(&as_it, NULL, (void **)&vtd_as)) { + if ((pci_bus_num(vtd_as->bus) == bus_n) && + (vtd_as->devfn & mask) == (devfn & mask)) { + trace_vtd_inv_desc_cc_device(bus_n, VTD_PCI_SLOT(vtd_as->devfn), + VTD_PCI_FUNC(vtd_as->devfn)); + vtd_iommu_lock(s); + vtd_as->context_cache_entry.context_cache_gen = 0; + vtd_iommu_unlock(s); + /* + * Do switch address space when needed, in case if the + * device passthrough bit is switched. + */ + vtd_switch_address_space(vtd_as); + /* + * So a device is moving out of (or moving into) a + * domain, resync the shadow page table. + * This won't bring bad even if we have no such + * notifier registered - the IOMMU notification + * framework will skip MAP notifications if that + * happened. + */ + vtd_sync_shadow_page_table(vtd_as); } } } @@ -2473,18 +2476,13 @@ static bool vtd_process_device_iotlb_desc(IntelIOMMUState *s, { VTDAddressSpace *vtd_dev_as; IOMMUTLBEvent event; - struct VTDBus *vtd_bus; hwaddr addr; uint64_t sz; uint16_t sid; - uint8_t devfn; bool size; - uint8_t bus_num; addr = VTD_INV_DESC_DEVICE_IOTLB_ADDR(inv_desc->hi); sid = VTD_INV_DESC_DEVICE_IOTLB_SID(inv_desc->lo); - devfn = sid & 0xff; - bus_num = sid >> 8; size = VTD_INV_DESC_DEVICE_IOTLB_SIZE(inv_desc->hi); if ((inv_desc->lo & VTD_INV_DESC_DEVICE_IOTLB_RSVD_LO) || @@ -2495,12 +2493,11 @@ static bool vtd_process_device_iotlb_desc(IntelIOMMUState *s, return false; } - vtd_bus = vtd_find_as_from_bus_num(s, bus_num); - if (!vtd_bus) { - goto done; - } - - vtd_dev_as = vtd_bus->dev_as[devfn]; + /* + * Using sid is OK since the guest should have finished the + * initialization of both the bus and device. + */ + vtd_dev_as = vtd_get_as_by_sid(s, sid); if (!vtd_dev_as) { goto done; } @@ -3427,27 +3424,27 @@ static const MemoryRegionOps vtd_mem_ir_ops = { VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn) { - uintptr_t key = (uintptr_t)bus; - VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key); + /* + * We can't simply use sid here since the bus number might not be + * initialized by the guest. + */ + struct vtd_as_key key = { + .bus = bus, + .devfn = devfn, + }; VTDAddressSpace *vtd_dev_as; char name[128]; - if (!vtd_bus) { - uintptr_t *new_key = g_malloc(sizeof(*new_key)); - *new_key = (uintptr_t)bus; - /* No corresponding free() */ - vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * \ - PCI_DEVFN_MAX); - vtd_bus->bus = bus; - g_hash_table_insert(s->vtd_as_by_busptr, new_key, vtd_bus); - } + vtd_dev_as = g_hash_table_lookup(s->vtd_address_spaces, &key); + if (!vtd_dev_as) { + struct vtd_as_key *new_key = g_malloc(sizeof(*new_key)); - vtd_dev_as = vtd_bus->dev_as[devfn]; + new_key->bus = bus; + new_key->devfn = devfn; - if (!vtd_dev_as) { snprintf(name, sizeof(name), "vtd-%02x.%x", PCI_SLOT(devfn), PCI_FUNC(devfn)); - vtd_bus->dev_as[devfn] = vtd_dev_as = g_new0(VTDAddressSpace, 1); + vtd_dev_as = g_new0(VTDAddressSpace, 1); vtd_dev_as->bus = bus; vtd_dev_as->devfn = (uint8_t)devfn; @@ -3503,6 +3500,8 @@ VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn) &vtd_dev_as->nodmar, 0); vtd_switch_address_space(vtd_dev_as); + + g_hash_table_insert(s->vtd_address_spaces, new_key, vtd_dev_as); } return vtd_dev_as; } @@ -3881,7 +3880,6 @@ static void vtd_realize(DeviceState *dev, Error **errp) QLIST_INIT(&s->vtd_as_with_notifiers); qemu_mutex_init(&s->iommu_lock); - memset(s->vtd_as_by_bus_num, 0, sizeof(s->vtd_as_by_bus_num)); memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s, "intel_iommu", DMAR_REG_SIZE); @@ -3903,8 +3901,8 @@ static void vtd_realize(DeviceState *dev, Error **errp) /* No corresponding destroy */ s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal, g_free, g_free); - s->vtd_as_by_busptr = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal, - g_free, g_free); + s->vtd_address_spaces = g_hash_table_new_full(vtd_as_hash, vtd_as_equal, + g_free, g_free); vtd_init(s); sysbus_mmio_map(SYS_BUS_DEVICE(s), 0, Q35_HOST_BRIDGE_IOMMU_ADDR); pci_setup_iommu(bus, vtd_host_dma_iommu, dev); diff --git a/include/hw/i386/intel_iommu.h b/include/hw/i386/intel_iommu.h index 67653b0f9b..e49fff2a6c 100644 --- a/include/hw/i386/intel_iommu.h +++ b/include/hw/i386/intel_iommu.h @@ -58,7 +58,6 @@ typedef struct VTDContextEntry VTDContextEntry; typedef struct VTDContextCacheEntry VTDContextCacheEntry; typedef struct VTDAddressSpace VTDAddressSpace; typedef struct VTDIOTLBEntry VTDIOTLBEntry; -typedef struct VTDBus VTDBus; typedef union VTD_IR_TableEntry VTD_IR_TableEntry; typedef union VTD_IR_MSIAddress VTD_IR_MSIAddress; typedef struct VTDPASIDDirEntry VTDPASIDDirEntry; @@ -111,12 +110,6 @@ struct VTDAddressSpace { IOVATree *iova_tree; /* Traces mapped IOVA ranges */ }; -struct VTDBus { - PCIBus* bus; /* A reference to the bus to provide translation for */ - /* A table of VTDAddressSpace objects indexed by devfn */ - VTDAddressSpace *dev_as[]; -}; - struct VTDIOTLBEntry { uint64_t gfn; uint16_t domain_id; @@ -253,8 +246,8 @@ struct IntelIOMMUState { uint32_t context_cache_gen; /* Should be in [1,MAX] */ GHashTable *iotlb; /* IOTLB */ - GHashTable *vtd_as_by_busptr; /* VTDBus objects indexed by PCIBus* reference */ - VTDBus *vtd_as_by_bus_num[VTD_PCI_BUS_MAX]; /* VTDBus objects indexed by bus number */ + GHashTable *vtd_address_spaces; /* VTD address spaces */ + VTDAddressSpace *vtd_as_cache[VTD_PCI_BUS_MAX]; /* VTD address space cache */ /* list of registered notifiers */ QLIST_HEAD(, VTDAddressSpace) vtd_as_with_notifiers; -- cgit v1.2.3-55-g7522 From 1b2b12376c8a513a0c7b5e3b8ea702038d3d7db5 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 28 Oct 2022 14:14:36 +0800 Subject: intel-iommu: PASID support This patch introduce ECAP_PASID via "x-pasid-mode". Based on the existing support for scalable mode, we need to implement the following missing parts: 1) tag VTDAddressSpace with PASID and support IOMMU/DMA translation with PASID 2) tag IOTLB with PASID 3) PASID cache and its flush 4) PASID based IOTLB invalidation For simplicity PASID cache is not implemented so we can simply implement the PASID cache flush as a no and leave it to be implemented in the future. For PASID based IOTLB invalidation, since we haven't had L1 stage support, the PASID based IOTLB invalidation is not implemented yet. For PASID based device IOTLB invalidation, it requires the support for vhost so we forbid enabling device IOTLB when PASID is enabled now. Those work could be done in the future. Note that though PASID based IOMMU translation is ready but no device can issue PASID DMA right now. In this case, PCI_NO_PASID is used as PASID to identify the address without PASID. vtd_find_add_as() has been extended to provision address space with PASID which could be utilized by the future extension of PCI core to allow device model to use PASID based DMA translation. This feature would be useful for: 1) prototyping PASID support for devices like virtio 2) future vPASID work 3) future PRS and vSVA work Reviewed-by: Peter Xu Signed-off-by: Jason Wang Message-Id: <20221028061436.30093-5-jasowang@redhat.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- hw/i386/intel_iommu.c | 418 +++++++++++++++++++++++++++++++---------- hw/i386/intel_iommu_internal.h | 16 +- hw/i386/trace-events | 2 + include/hw/i386/intel_iommu.h | 7 +- include/hw/pci/pci_bus.h | 2 + 5 files changed, 341 insertions(+), 104 deletions(-) (limited to 'include') diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c index 259f720c7c..a08ee85edf 100644 --- a/hw/i386/intel_iommu.c +++ b/hw/i386/intel_iommu.c @@ -58,6 +58,14 @@ struct vtd_as_key { PCIBus *bus; uint8_t devfn; + uint32_t pasid; +}; + +struct vtd_iotlb_key { + uint64_t gfn; + uint32_t pasid; + uint32_t level; + uint16_t sid; }; static void vtd_address_space_refresh_all(IntelIOMMUState *s); @@ -199,14 +207,24 @@ static inline gboolean vtd_as_has_map_notifier(VTDAddressSpace *as) } /* GHashTable functions */ -static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2) +static gboolean vtd_iotlb_equal(gconstpointer v1, gconstpointer v2) { - return *((const uint64_t *)v1) == *((const uint64_t *)v2); + const struct vtd_iotlb_key *key1 = v1; + const struct vtd_iotlb_key *key2 = v2; + + return key1->sid == key2->sid && + key1->pasid == key2->pasid && + key1->level == key2->level && + key1->gfn == key2->gfn; } -static guint vtd_uint64_hash(gconstpointer v) +static guint vtd_iotlb_hash(gconstpointer v) { - return (guint)*(const uint64_t *)v; + const struct vtd_iotlb_key *key = v; + + return key->gfn | ((key->sid) << VTD_IOTLB_SID_SHIFT) | + (key->level) << VTD_IOTLB_LVL_SHIFT | + (key->pasid) << VTD_IOTLB_PASID_SHIFT; } static gboolean vtd_as_equal(gconstpointer v1, gconstpointer v2) @@ -214,7 +232,8 @@ static gboolean vtd_as_equal(gconstpointer v1, gconstpointer v2) const struct vtd_as_key *key1 = v1; const struct vtd_as_key *key2 = v2; - return (key1->bus == key2->bus) && (key1->devfn == key2->devfn); + return (key1->bus == key2->bus) && (key1->devfn == key2->devfn) && + (key1->pasid == key2->pasid); } /* @@ -302,13 +321,6 @@ static void vtd_reset_caches(IntelIOMMUState *s) vtd_iommu_unlock(s); } -static uint64_t vtd_get_iotlb_key(uint64_t gfn, uint16_t source_id, - uint32_t level) -{ - return gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT) | - ((uint64_t)(level) << VTD_IOTLB_LVL_SHIFT); -} - static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level) { return (addr & vtd_slpt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K; @@ -316,15 +328,17 @@ static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level) /* Must be called with IOMMU lock held */ static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id, - hwaddr addr) + uint32_t pasid, hwaddr addr) { + struct vtd_iotlb_key key; VTDIOTLBEntry *entry; - uint64_t key; int level; for (level = VTD_SL_PT_LEVEL; level < VTD_SL_PML4_LEVEL; level++) { - key = vtd_get_iotlb_key(vtd_get_iotlb_gfn(addr, level), - source_id, level); + key.gfn = vtd_get_iotlb_gfn(addr, level); + key.level = level; + key.sid = source_id; + key.pasid = pasid; entry = g_hash_table_lookup(s->iotlb, &key); if (entry) { goto out; @@ -338,10 +352,11 @@ out: /* Must be with IOMMU lock held */ static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id, uint16_t domain_id, hwaddr addr, uint64_t slpte, - uint8_t access_flags, uint32_t level) + uint8_t access_flags, uint32_t level, + uint32_t pasid) { VTDIOTLBEntry *entry = g_malloc(sizeof(*entry)); - uint64_t *key = g_malloc(sizeof(*key)); + struct vtd_iotlb_key *key = g_malloc(sizeof(*key)); uint64_t gfn = vtd_get_iotlb_gfn(addr, level); trace_vtd_iotlb_page_update(source_id, addr, slpte, domain_id); @@ -355,7 +370,13 @@ static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id, entry->slpte = slpte; entry->access_flags = access_flags; entry->mask = vtd_slpt_level_page_mask(level); - *key = vtd_get_iotlb_key(gfn, source_id, level); + entry->pasid = pasid; + + key->gfn = gfn; + key->sid = source_id; + key->level = level; + key->pasid = pasid; + g_hash_table_replace(s->iotlb, key, entry); } @@ -448,7 +469,8 @@ static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index) /* Must not update F field now, should be done later */ static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index, uint16_t source_id, hwaddr addr, - VTDFaultReason fault, bool is_write) + VTDFaultReason fault, bool is_write, + bool is_pasid, uint32_t pasid) { uint64_t hi = 0, lo; hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); @@ -456,7 +478,8 @@ static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index, assert(index < DMAR_FRCD_REG_NR); lo = VTD_FRCD_FI(addr); - hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault); + hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault) | + VTD_FRCD_PV(pasid) | VTD_FRCD_PP(is_pasid); if (!is_write) { hi |= VTD_FRCD_T; } @@ -487,7 +510,8 @@ static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id) /* Log and report an DMAR (address translation) fault to software */ static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id, hwaddr addr, VTDFaultReason fault, - bool is_write) + bool is_write, bool is_pasid, + uint32_t pasid) { uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); @@ -514,7 +538,8 @@ static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id, return; } - vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, is_write); + vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, + is_write, is_pasid, pasid); if (fsts_reg & VTD_FSTS_PPF) { error_report_once("There are pending faults already, " @@ -819,13 +844,15 @@ static int vtd_get_pe_from_pasid_table(IntelIOMMUState *s, static int vtd_ce_get_rid2pasid_entry(IntelIOMMUState *s, VTDContextEntry *ce, - VTDPASIDEntry *pe) + VTDPASIDEntry *pe, + uint32_t pasid) { - uint32_t pasid; dma_addr_t pasid_dir_base; int ret = 0; - pasid = VTD_CE_GET_RID2PASID(ce); + if (pasid == PCI_NO_PASID) { + pasid = VTD_CE_GET_RID2PASID(ce); + } pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce); ret = vtd_get_pe_from_pasid_table(s, pasid_dir_base, pasid, pe); @@ -834,15 +861,17 @@ static int vtd_ce_get_rid2pasid_entry(IntelIOMMUState *s, static int vtd_ce_get_pasid_fpd(IntelIOMMUState *s, VTDContextEntry *ce, - bool *pe_fpd_set) + bool *pe_fpd_set, + uint32_t pasid) { int ret; - uint32_t pasid; dma_addr_t pasid_dir_base; VTDPASIDDirEntry pdire; VTDPASIDEntry pe; - pasid = VTD_CE_GET_RID2PASID(ce); + if (pasid == PCI_NO_PASID) { + pasid = VTD_CE_GET_RID2PASID(ce); + } pasid_dir_base = VTD_CE_GET_PASID_DIR_TABLE(ce); /* @@ -888,12 +917,13 @@ static inline uint32_t vtd_ce_get_level(VTDContextEntry *ce) } static uint32_t vtd_get_iova_level(IntelIOMMUState *s, - VTDContextEntry *ce) + VTDContextEntry *ce, + uint32_t pasid) { VTDPASIDEntry pe; if (s->root_scalable) { - vtd_ce_get_rid2pasid_entry(s, ce, &pe); + vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); return VTD_PE_GET_LEVEL(&pe); } @@ -906,12 +936,13 @@ static inline uint32_t vtd_ce_get_agaw(VTDContextEntry *ce) } static uint32_t vtd_get_iova_agaw(IntelIOMMUState *s, - VTDContextEntry *ce) + VTDContextEntry *ce, + uint32_t pasid) { VTDPASIDEntry pe; if (s->root_scalable) { - vtd_ce_get_rid2pasid_entry(s, ce, &pe); + vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); return 30 + ((pe.val[0] >> 2) & VTD_SM_PASID_ENTRY_AW) * 9; } @@ -953,31 +984,33 @@ static inline bool vtd_ce_type_check(X86IOMMUState *x86_iommu, } static inline uint64_t vtd_iova_limit(IntelIOMMUState *s, - VTDContextEntry *ce, uint8_t aw) + VTDContextEntry *ce, uint8_t aw, + uint32_t pasid) { - uint32_t ce_agaw = vtd_get_iova_agaw(s, ce); + uint32_t ce_agaw = vtd_get_iova_agaw(s, ce, pasid); return 1ULL << MIN(ce_agaw, aw); } /* Return true if IOVA passes range check, otherwise false. */ static inline bool vtd_iova_range_check(IntelIOMMUState *s, uint64_t iova, VTDContextEntry *ce, - uint8_t aw) + uint8_t aw, uint32_t pasid) { /* * Check if @iova is above 2^X-1, where X is the minimum of MGAW * in CAP_REG and AW in context-entry. */ - return !(iova & ~(vtd_iova_limit(s, ce, aw) - 1)); + return !(iova & ~(vtd_iova_limit(s, ce, aw, pasid) - 1)); } static dma_addr_t vtd_get_iova_pgtbl_base(IntelIOMMUState *s, - VTDContextEntry *ce) + VTDContextEntry *ce, + uint32_t pasid) { VTDPASIDEntry pe; if (s->root_scalable) { - vtd_ce_get_rid2pasid_entry(s, ce, &pe); + vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); return pe.val[0] & VTD_SM_PASID_ENTRY_SLPTPTR; } @@ -1011,18 +1044,19 @@ static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level) static int vtd_iova_to_slpte(IntelIOMMUState *s, VTDContextEntry *ce, uint64_t iova, bool is_write, uint64_t *slptep, uint32_t *slpte_level, - bool *reads, bool *writes, uint8_t aw_bits) + bool *reads, bool *writes, uint8_t aw_bits, + uint32_t pasid) { - dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce); - uint32_t level = vtd_get_iova_level(s, ce); + dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce, pasid); + uint32_t level = vtd_get_iova_level(s, ce, pasid); uint32_t offset; uint64_t slpte; uint64_t access_right_check; uint64_t xlat, size; - if (!vtd_iova_range_check(s, iova, ce, aw_bits)) { - error_report_once("%s: detected IOVA overflow (iova=0x%" PRIx64 ")", - __func__, iova); + if (!vtd_iova_range_check(s, iova, ce, aw_bits, pasid)) { + error_report_once("%s: detected IOVA overflow (iova=0x%" PRIx64 "," + "pasid=0x%" PRIx32 ")", __func__, iova, pasid); return -VTD_FR_ADDR_BEYOND_MGAW; } @@ -1035,8 +1069,9 @@ static int vtd_iova_to_slpte(IntelIOMMUState *s, VTDContextEntry *ce, if (slpte == (uint64_t)-1) { error_report_once("%s: detected read error on DMAR slpte " - "(iova=0x%" PRIx64 ")", __func__, iova); - if (level == vtd_get_iova_level(s, ce)) { + "(iova=0x%" PRIx64 ", pasid=0x%" PRIx32 ")", + __func__, iova, pasid); + if (level == vtd_get_iova_level(s, ce, pasid)) { /* Invalid programming of context-entry */ return -VTD_FR_CONTEXT_ENTRY_INV; } else { @@ -1048,15 +1083,16 @@ static int vtd_iova_to_slpte(IntelIOMMUState *s, VTDContextEntry *ce, if (!(slpte & access_right_check)) { error_report_once("%s: detected slpte permission error " "(iova=0x%" PRIx64 ", level=0x%" PRIx32 ", " - "slpte=0x%" PRIx64 ", write=%d)", __func__, - iova, level, slpte, is_write); + "slpte=0x%" PRIx64 ", write=%d, pasid=0x%" + PRIx32 ")", __func__, iova, level, + slpte, is_write, pasid); return is_write ? -VTD_FR_WRITE : -VTD_FR_READ; } if (vtd_slpte_nonzero_rsvd(slpte, level)) { error_report_once("%s: detected splte reserve non-zero " "iova=0x%" PRIx64 ", level=0x%" PRIx32 - "slpte=0x%" PRIx64 ")", __func__, iova, - level, slpte); + "slpte=0x%" PRIx64 ", pasid=0x%" PRIX32 ")", + __func__, iova, level, slpte, pasid); return -VTD_FR_PAGING_ENTRY_RSVD; } @@ -1084,9 +1120,10 @@ static int vtd_iova_to_slpte(IntelIOMMUState *s, VTDContextEntry *ce, error_report_once("%s: xlat address is in interrupt range " "(iova=0x%" PRIx64 ", level=0x%" PRIx32 ", " "slpte=0x%" PRIx64 ", write=%d, " - "xlat=0x%" PRIx64 ", size=0x%" PRIx64 ")", + "xlat=0x%" PRIx64 ", size=0x%" PRIx64 ", " + "pasid=0x%" PRIx32 ")", __func__, iova, level, slpte, is_write, - xlat, size); + xlat, size, pasid); return s->scalable_mode ? -VTD_FR_SM_INTERRUPT_ADDR : -VTD_FR_INTERRUPT_ADDR; } @@ -1300,18 +1337,19 @@ next: */ static int vtd_page_walk(IntelIOMMUState *s, VTDContextEntry *ce, uint64_t start, uint64_t end, - vtd_page_walk_info *info) + vtd_page_walk_info *info, + uint32_t pasid) { - dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce); - uint32_t level = vtd_get_iova_level(s, ce); + dma_addr_t addr = vtd_get_iova_pgtbl_base(s, ce, pasid); + uint32_t level = vtd_get_iova_level(s, ce, pasid); - if (!vtd_iova_range_check(s, start, ce, info->aw)) { + if (!vtd_iova_range_check(s, start, ce, info->aw, pasid)) { return -VTD_FR_ADDR_BEYOND_MGAW; } - if (!vtd_iova_range_check(s, end, ce, info->aw)) { + if (!vtd_iova_range_check(s, end, ce, info->aw, pasid)) { /* Fix end so that it reaches the maximum */ - end = vtd_iova_limit(s, ce, info->aw); + end = vtd_iova_limit(s, ce, info->aw, pasid); } return vtd_page_walk_level(addr, start, end, level, true, true, info); @@ -1379,7 +1417,7 @@ static int vtd_ce_rid2pasid_check(IntelIOMMUState *s, * has valid rid2pasid setting, which includes valid * rid2pasid field and corresponding pasid entry setting */ - return vtd_ce_get_rid2pasid_entry(s, ce, &pe); + return vtd_ce_get_rid2pasid_entry(s, ce, &pe, PCI_NO_PASID); } /* Map a device to its corresponding domain (context-entry) */ @@ -1462,12 +1500,13 @@ static int vtd_sync_shadow_page_hook(IOMMUTLBEvent *event, } static uint16_t vtd_get_domain_id(IntelIOMMUState *s, - VTDContextEntry *ce) + VTDContextEntry *ce, + uint32_t pasid) { VTDPASIDEntry pe; if (s->root_scalable) { - vtd_ce_get_rid2pasid_entry(s, ce, &pe); + vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); return VTD_SM_PASID_ENTRY_DID(pe.val[1]); } @@ -1485,10 +1524,10 @@ static int vtd_sync_shadow_page_table_range(VTDAddressSpace *vtd_as, .notify_unmap = true, .aw = s->aw_bits, .as = vtd_as, - .domain_id = vtd_get_domain_id(s, ce), + .domain_id = vtd_get_domain_id(s, ce, vtd_as->pasid), }; - return vtd_page_walk(s, ce, addr, addr + size, &info); + return vtd_page_walk(s, ce, addr, addr + size, &info, vtd_as->pasid); } static int vtd_sync_shadow_page_table(VTDAddressSpace *vtd_as) @@ -1532,13 +1571,14 @@ static int vtd_sync_shadow_page_table(VTDAddressSpace *vtd_as) * 1st-level translation or 2nd-level translation, it depends * on PGTT setting. */ -static bool vtd_dev_pt_enabled(IntelIOMMUState *s, VTDContextEntry *ce) +static bool vtd_dev_pt_enabled(IntelIOMMUState *s, VTDContextEntry *ce, + uint32_t pasid) { VTDPASIDEntry pe; int ret; if (s->root_scalable) { - ret = vtd_ce_get_rid2pasid_entry(s, ce, &pe); + ret = vtd_ce_get_rid2pasid_entry(s, ce, &pe, pasid); if (ret) { /* * This error is guest triggerable. We should assumt PT @@ -1572,19 +1612,20 @@ static bool vtd_as_pt_enabled(VTDAddressSpace *as) return false; } - return vtd_dev_pt_enabled(s, &ce); + return vtd_dev_pt_enabled(s, &ce, as->pasid); } /* Return whether the device is using IOMMU translation. */ static bool vtd_switch_address_space(VTDAddressSpace *as) { - bool use_iommu; + bool use_iommu, pt; /* Whether we need to take the BQL on our own */ bool take_bql = !qemu_mutex_iothread_locked(); assert(as); use_iommu = as->iommu_state->dmar_enabled && !vtd_as_pt_enabled(as); + pt = as->iommu_state->dmar_enabled && vtd_as_pt_enabled(as); trace_vtd_switch_address_space(pci_bus_num(as->bus), VTD_PCI_SLOT(as->devfn), @@ -1604,11 +1645,53 @@ static bool vtd_switch_address_space(VTDAddressSpace *as) if (use_iommu) { memory_region_set_enabled(&as->nodmar, false); memory_region_set_enabled(MEMORY_REGION(&as->iommu), true); + /* + * vt-d spec v3.4 3.14: + * + * """ + * Requests-with-PASID with input address in range 0xFEEx_xxxx + * are translated normally like any other request-with-PASID + * through DMA-remapping hardware. + * """ + * + * Need to disable ir for as with PASID. + */ + if (as->pasid != PCI_NO_PASID) { + memory_region_set_enabled(&as->iommu_ir, false); + } else { + memory_region_set_enabled(&as->iommu_ir, true); + } } else { memory_region_set_enabled(MEMORY_REGION(&as->iommu), false); memory_region_set_enabled(&as->nodmar, true); } + /* + * vtd-spec v3.4 3.14: + * + * """ + * Requests-with-PASID with input address in range 0xFEEx_xxxx are + * translated normally like any other request-with-PASID through + * DMA-remapping hardware. However, if such a request is processed + * using pass-through translation, it will be blocked as described + * in the paragraph below. + * + * Software must not program paging-structure entries to remap any + * address to the interrupt address range. Untranslated requests + * and translation requests that result in an address in the + * interrupt range will be blocked with condition code LGN.4 or + * SGN.8. + * """ + * + * We enable per as memory region (iommu_ir_fault) for catching + * the tranlsation for interrupt range through PASID + PT. + */ + if (pt && as->pasid != PCI_NO_PASID) { + memory_region_set_enabled(&as->iommu_ir_fault, true); + } else { + memory_region_set_enabled(&as->iommu_ir_fault, false); + } + if (take_bql) { qemu_mutex_unlock_iothread(); } @@ -1709,12 +1792,15 @@ static void vtd_report_fault(IntelIOMMUState *s, int err, bool is_fpd_set, uint16_t source_id, hwaddr addr, - bool is_write) + bool is_write, + bool is_pasid, + uint32_t pasid) { if (is_fpd_set && vtd_is_qualified_fault(err)) { trace_vtd_fault_disabled(); } else { - vtd_report_dmar_fault(s, source_id, addr, err, is_write); + vtd_report_dmar_fault(s, source_id, addr, err, is_write, + is_pasid, pasid); } } @@ -1739,13 +1825,14 @@ static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, uint8_t bus_num = pci_bus_num(bus); VTDContextCacheEntry *cc_entry; uint64_t slpte, page_mask; - uint32_t level; + uint32_t level, pasid = vtd_as->pasid; uint16_t source_id = PCI_BUILD_BDF(bus_num, devfn); int ret_fr; bool is_fpd_set = false; bool reads = true; bool writes = true; uint8_t access_flags; + bool rid2pasid = (pasid == PCI_NO_PASID) && s->root_scalable; VTDIOTLBEntry *iotlb_entry; /* @@ -1758,15 +1845,17 @@ static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, cc_entry = &vtd_as->context_cache_entry; - /* Try to fetch slpte form IOTLB */ - iotlb_entry = vtd_lookup_iotlb(s, source_id, addr); - if (iotlb_entry) { - trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->slpte, - iotlb_entry->domain_id); - slpte = iotlb_entry->slpte; - access_flags = iotlb_entry->access_flags; - page_mask = iotlb_entry->mask; - goto out; + /* Try to fetch slpte form IOTLB, we don't need RID2PASID logic */ + if (!rid2pasid) { + iotlb_entry = vtd_lookup_iotlb(s, source_id, pasid, addr); + if (iotlb_entry) { + trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->slpte, + iotlb_entry->domain_id); + slpte = iotlb_entry->slpte; + access_flags = iotlb_entry->access_flags; + page_mask = iotlb_entry->mask; + goto out; + } } /* Try to fetch context-entry from cache first */ @@ -1777,10 +1866,11 @@ static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, ce = cc_entry->context_entry; is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; if (!is_fpd_set && s->root_scalable) { - ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set); + ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, pasid); if (ret_fr) { vtd_report_fault(s, -ret_fr, is_fpd_set, - source_id, addr, is_write); + source_id, addr, is_write, + false, 0); goto error; } } @@ -1788,11 +1878,12 @@ static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce); is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; if (!ret_fr && !is_fpd_set && s->root_scalable) { - ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set); + ret_fr = vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, pasid); } if (ret_fr) { vtd_report_fault(s, -ret_fr, is_fpd_set, - source_id, addr, is_write); + source_id, addr, is_write, + false, 0); goto error; } /* Update context-cache */ @@ -1803,11 +1894,15 @@ static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, cc_entry->context_cache_gen = s->context_cache_gen; } + if (rid2pasid) { + pasid = VTD_CE_GET_RID2PASID(&ce); + } + /* * We don't need to translate for pass-through context entries. * Also, let's ignore IOTLB caching as well for PT devices. */ - if (vtd_dev_pt_enabled(s, &ce)) { + if (vtd_dev_pt_enabled(s, &ce, pasid)) { entry->iova = addr & VTD_PAGE_MASK_4K; entry->translated_addr = entry->iova; entry->addr_mask = ~VTD_PAGE_MASK_4K; @@ -1828,18 +1923,31 @@ static bool vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, return true; } + /* Try to fetch slpte form IOTLB for RID2PASID slow path */ + if (rid2pasid) { + iotlb_entry = vtd_lookup_iotlb(s, source_id, pasid, addr); + if (iotlb_entry) { + trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->slpte, + iotlb_entry->domain_id); + slpte = iotlb_entry->slpte; + access_flags = iotlb_entry->access_flags; + page_mask = iotlb_entry->mask; + goto out; + } + } + ret_fr = vtd_iova_to_slpte(s, &ce, addr, is_write, &slpte, &level, - &reads, &writes, s->aw_bits); + &reads, &writes, s->aw_bits, pasid); if (ret_fr) { vtd_report_fault(s, -ret_fr, is_fpd_set, source_id, - addr, is_write); + addr, is_write, pasid != PCI_NO_PASID, pasid); goto error; } page_mask = vtd_slpt_level_page_mask(level); access_flags = IOMMU_ACCESS_FLAG(reads, writes); - vtd_update_iotlb(s, source_id, vtd_get_domain_id(s, &ce), addr, slpte, - access_flags, level); + vtd_update_iotlb(s, source_id, vtd_get_domain_id(s, &ce, pasid), + addr, slpte, access_flags, level, pasid); out: vtd_iommu_unlock(s); entry->iova = addr & page_mask; @@ -2031,7 +2139,7 @@ static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id) QLIST_FOREACH(vtd_as, &s->vtd_as_with_notifiers, next) { if (!vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), vtd_as->devfn, &ce) && - domain_id == vtd_get_domain_id(s, &ce)) { + domain_id == vtd_get_domain_id(s, &ce, vtd_as->pasid)) { vtd_sync_shadow_page_table(vtd_as); } } @@ -2039,7 +2147,7 @@ static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id) static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s, uint16_t domain_id, hwaddr addr, - uint8_t am) + uint8_t am, uint32_t pasid) { VTDAddressSpace *vtd_as; VTDContextEntry ce; @@ -2047,9 +2155,12 @@ static void vtd_iotlb_page_invalidate_notify(IntelIOMMUState *s, hwaddr size = (1 << am) * VTD_PAGE_SIZE; QLIST_FOREACH(vtd_as, &(s->vtd_as_with_notifiers), next) { + if (pasid != PCI_NO_PASID && pasid != vtd_as->pasid) { + continue; + } ret = vtd_dev_to_context_entry(s, pci_bus_num(vtd_as->bus), vtd_as->devfn, &ce); - if (!ret && domain_id == vtd_get_domain_id(s, &ce)) { + if (!ret && domain_id == vtd_get_domain_id(s, &ce, vtd_as->pasid)) { if (vtd_as_has_map_notifier(vtd_as)) { /* * As long as we have MAP notifications registered in @@ -2093,7 +2204,7 @@ static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id, vtd_iommu_lock(s); g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info); vtd_iommu_unlock(s); - vtd_iotlb_page_invalidate_notify(s, domain_id, addr, am); + vtd_iotlb_page_invalidate_notify(s, domain_id, addr, am, PCI_NO_PASID); } /* Flush IOTLB @@ -3162,6 +3273,7 @@ static Property vtd_properties[] = { DEFINE_PROP_BOOL("caching-mode", IntelIOMMUState, caching_mode, FALSE), DEFINE_PROP_BOOL("x-scalable-mode", IntelIOMMUState, scalable_mode, FALSE), DEFINE_PROP_BOOL("snoop-control", IntelIOMMUState, snoop_control, false), + DEFINE_PROP_BOOL("x-pasid-mode", IntelIOMMUState, pasid, false), DEFINE_PROP_BOOL("dma-drain", IntelIOMMUState, dma_drain, true), DEFINE_PROP_BOOL("dma-translation", IntelIOMMUState, dma_translation, true), DEFINE_PROP_END_OF_LIST(), @@ -3436,7 +3548,64 @@ static const MemoryRegionOps vtd_mem_ir_ops = { }, }; -VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn) +static void vtd_report_ir_illegal_access(VTDAddressSpace *vtd_as, + hwaddr addr, bool is_write) +{ + IntelIOMMUState *s = vtd_as->iommu_state; + uint8_t bus_n = pci_bus_num(vtd_as->bus); + uint16_t sid = PCI_BUILD_BDF(bus_n, vtd_as->devfn); + bool is_fpd_set = false; + VTDContextEntry ce; + + assert(vtd_as->pasid != PCI_NO_PASID); + + /* Try out best to fetch FPD, we can't do anything more */ + if (vtd_dev_to_context_entry(s, bus_n, vtd_as->devfn, &ce) == 0) { + is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; + if (!is_fpd_set && s->root_scalable) { + vtd_ce_get_pasid_fpd(s, &ce, &is_fpd_set, vtd_as->pasid); + } + } + + vtd_report_fault(s, VTD_FR_SM_INTERRUPT_ADDR, + is_fpd_set, sid, addr, is_write, + true, vtd_as->pasid); +} + +static MemTxResult vtd_mem_ir_fault_read(void *opaque, hwaddr addr, + uint64_t *data, unsigned size, + MemTxAttrs attrs) +{ + vtd_report_ir_illegal_access(opaque, addr, false); + + return MEMTX_ERROR; +} + +static MemTxResult vtd_mem_ir_fault_write(void *opaque, hwaddr addr, + uint64_t value, unsigned size, + MemTxAttrs attrs) +{ + vtd_report_ir_illegal_access(opaque, addr, true); + + return MEMTX_ERROR; +} + +static const MemoryRegionOps vtd_mem_ir_fault_ops = { + .read_with_attrs = vtd_mem_ir_fault_read, + .write_with_attrs = vtd_mem_ir_fault_write, + .endianness = DEVICE_LITTLE_ENDIAN, + .impl = { + .min_access_size = 1, + .max_access_size = 8, + }, + .valid = { + .min_access_size = 1, + .max_access_size = 8, + }, +}; + +VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, + int devfn, unsigned int pasid) { /* * We can't simply use sid here since the bus number might not be @@ -3445,6 +3614,7 @@ VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn) struct vtd_as_key key = { .bus = bus, .devfn = devfn, + .pasid = pasid, }; VTDAddressSpace *vtd_dev_as; char name[128]; @@ -3455,13 +3625,21 @@ VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn) new_key->bus = bus; new_key->devfn = devfn; + new_key->pasid = pasid; + + if (pasid == PCI_NO_PASID) { + snprintf(name, sizeof(name), "vtd-%02x.%x", PCI_SLOT(devfn), + PCI_FUNC(devfn)); + } else { + snprintf(name, sizeof(name), "vtd-%02x.%x-pasid-%x", PCI_SLOT(devfn), + PCI_FUNC(devfn), pasid); + } - snprintf(name, sizeof(name), "vtd-%02x.%x", PCI_SLOT(devfn), - PCI_FUNC(devfn)); vtd_dev_as = g_new0(VTDAddressSpace, 1); vtd_dev_as->bus = bus; vtd_dev_as->devfn = (uint8_t)devfn; + vtd_dev_as->pasid = pasid; vtd_dev_as->iommu_state = s; vtd_dev_as->context_cache_entry.context_cache_gen = 0; vtd_dev_as->iova_tree = iova_tree_new(); @@ -3502,6 +3680,24 @@ VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn) VTD_INTERRUPT_ADDR_FIRST, &vtd_dev_as->iommu_ir, 1); + /* + * This region is used for catching fault to access interrupt + * range via passthrough + PASID. See also + * vtd_switch_address_space(). We can't use alias since we + * need to know the sid which is valid for MSI who uses + * bus_master_as (see msi_send_message()). + */ + memory_region_init_io(&vtd_dev_as->iommu_ir_fault, OBJECT(s), + &vtd_mem_ir_fault_ops, vtd_dev_as, "vtd-no-ir", + VTD_INTERRUPT_ADDR_SIZE); + /* + * Hook to root since when PT is enabled vtd_dev_as->iommu + * will be disabled. + */ + memory_region_add_subregion_overlap(MEMORY_REGION(&vtd_dev_as->root), + VTD_INTERRUPT_ADDR_FIRST, + &vtd_dev_as->iommu_ir_fault, 2); + /* * Hook both the containers under the root container, we * switch between DMAR & noDMAR by enable/disable @@ -3622,7 +3818,7 @@ static void vtd_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n) "legacy mode", bus_n, PCI_SLOT(vtd_as->devfn), PCI_FUNC(vtd_as->devfn), - vtd_get_domain_id(s, &ce), + vtd_get_domain_id(s, &ce, vtd_as->pasid), ce.hi, ce.lo); if (vtd_as_has_map_notifier(vtd_as)) { /* This is required only for MAP typed notifiers */ @@ -3632,10 +3828,10 @@ static void vtd_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n) .notify_unmap = false, .aw = s->aw_bits, .as = vtd_as, - .domain_id = vtd_get_domain_id(s, &ce), + .domain_id = vtd_get_domain_id(s, &ce, vtd_as->pasid), }; - vtd_page_walk(s, &ce, 0, ~0ULL, &info); + vtd_page_walk(s, &ce, 0, ~0ULL, &info, vtd_as->pasid); } } else { trace_vtd_replay_ce_invalid(bus_n, PCI_SLOT(vtd_as->devfn), @@ -3735,6 +3931,10 @@ static void vtd_init(IntelIOMMUState *s) s->ecap |= VTD_ECAP_SC; } + if (s->pasid) { + s->ecap |= VTD_ECAP_PASID; + } + vtd_reset_caches(s); /* Define registers with default values and bit semantics */ @@ -3808,7 +4008,7 @@ static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn) assert(0 <= devfn && devfn < PCI_DEVFN_MAX); - vtd_as = vtd_find_add_as(s, bus, devfn); + vtd_as = vtd_find_add_as(s, bus, devfn, PCI_NO_PASID); return &vtd_as->as; } @@ -3851,6 +4051,11 @@ static bool vtd_decide_config(IntelIOMMUState *s, Error **errp) return false; } + if (s->pasid && !s->scalable_mode) { + error_setg(errp, "Need to set scalable mode for PASID"); + return false; + } + return true; } @@ -3887,6 +4092,17 @@ static void vtd_realize(DeviceState *dev, Error **errp) X86MachineState *x86ms = X86_MACHINE(ms); PCIBus *bus = pcms->bus; IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); + X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s); + + if (s->pasid && x86_iommu->dt_supported) { + /* + * PASID-based-Device-TLB Invalidate Descriptor is not + * implemented and it requires support from vhost layer which + * needs to be implemented in the future. + */ + error_setg(errp, "PASID based device IOTLB is not supported"); + return; + } if (!vtd_decide_config(s, errp)) { return; @@ -3913,7 +4129,7 @@ static void vtd_realize(DeviceState *dev, Error **errp) sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem); /* No corresponding destroy */ - s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal, + s->iotlb = g_hash_table_new_full(vtd_iotlb_hash, vtd_iotlb_equal, g_free, g_free); s->vtd_address_spaces = g_hash_table_new_full(vtd_as_hash, vtd_as_equal, g_free, g_free); diff --git a/hw/i386/intel_iommu_internal.h b/hw/i386/intel_iommu_internal.h index 930ce61feb..f090e61e11 100644 --- a/hw/i386/intel_iommu_internal.h +++ b/hw/i386/intel_iommu_internal.h @@ -114,8 +114,9 @@ VTD_INTERRUPT_ADDR_FIRST + 1) /* The shift of source_id in the key of IOTLB hash table */ -#define VTD_IOTLB_SID_SHIFT 36 -#define VTD_IOTLB_LVL_SHIFT 52 +#define VTD_IOTLB_SID_SHIFT 20 +#define VTD_IOTLB_LVL_SHIFT 28 +#define VTD_IOTLB_PASID_SHIFT 30 #define VTD_IOTLB_MAX_SIZE 1024 /* Max size of the hash table */ /* IOTLB_REG */ @@ -191,6 +192,7 @@ #define VTD_ECAP_SC (1ULL << 7) #define VTD_ECAP_MHMV (15ULL << 20) #define VTD_ECAP_SRS (1ULL << 31) +#define VTD_ECAP_PASID (1ULL << 40) #define VTD_ECAP_SMTS (1ULL << 43) #define VTD_ECAP_SLTS (1ULL << 46) @@ -211,6 +213,8 @@ #define VTD_CAP_DRAIN_READ (1ULL << 55) #define VTD_CAP_DRAIN (VTD_CAP_DRAIN_READ | VTD_CAP_DRAIN_WRITE) #define VTD_CAP_CM (1ULL << 7) +#define VTD_PASID_ID_SHIFT 20 +#define VTD_PASID_ID_MASK ((1ULL << VTD_PASID_ID_SHIFT) - 1) /* Supported Adjusted Guest Address Widths */ #define VTD_CAP_SAGAW_SHIFT 8 @@ -262,6 +266,8 @@ #define VTD_FRCD_SID(val) ((val) & VTD_FRCD_SID_MASK) /* For the low 64-bit of 128-bit */ #define VTD_FRCD_FI(val) ((val) & ~0xfffULL) +#define VTD_FRCD_PV(val) (((val) & 0xffffULL) << 40) +#define VTD_FRCD_PP(val) (((val) & 0x1) << 31) /* DMA Remapping Fault Conditions */ typedef enum VTDFaultReason { @@ -379,6 +385,11 @@ typedef union VTDInvDesc VTDInvDesc; #define VTD_INV_DESC_IOTLB_AM(val) ((val) & 0x3fULL) #define VTD_INV_DESC_IOTLB_RSVD_LO 0xffffffff0000ff00ULL #define VTD_INV_DESC_IOTLB_RSVD_HI 0xf80ULL +#define VTD_INV_DESC_IOTLB_PASID_PASID (2ULL << 4) +#define VTD_INV_DESC_IOTLB_PASID_PAGE (3ULL << 4) +#define VTD_INV_DESC_IOTLB_PASID(val) (((val) >> 32) & VTD_PASID_ID_MASK) +#define VTD_INV_DESC_IOTLB_PASID_RSVD_LO 0xfff00000000001c0ULL +#define VTD_INV_DESC_IOTLB_PASID_RSVD_HI 0xf80ULL /* Mask for Device IOTLB Invalidate Descriptor */ #define VTD_INV_DESC_DEVICE_IOTLB_ADDR(val) ((val) & 0xfffffffffffff000ULL) @@ -413,6 +424,7 @@ typedef union VTDInvDesc VTDInvDesc; /* Information about page-selective IOTLB invalidate */ struct VTDIOTLBPageInvInfo { uint16_t domain_id; + uint32_t pasid; uint64_t addr; uint8_t mask; }; diff --git a/hw/i386/trace-events b/hw/i386/trace-events index e49814dd64..04fd71bfc4 100644 --- a/hw/i386/trace-events +++ b/hw/i386/trace-events @@ -12,6 +12,8 @@ vtd_inv_desc_cc_devices(uint16_t sid, uint16_t fmask) "context invalidate device vtd_inv_desc_iotlb_global(void) "iotlb invalidate global" vtd_inv_desc_iotlb_domain(uint16_t domain) "iotlb invalidate whole domain 0x%"PRIx16 vtd_inv_desc_iotlb_pages(uint16_t domain, uint64_t addr, uint8_t mask) "iotlb invalidate domain 0x%"PRIx16" addr 0x%"PRIx64" mask 0x%"PRIx8 +vtd_inv_desc_iotlb_pasid_pages(uint16_t domain, uint64_t addr, uint8_t mask, uint32_t pasid) "iotlb invalidate domain 0x%"PRIx16" addr 0x%"PRIx64" mask 0x%"PRIx8" pasid 0x%"PRIx32 +vtd_inv_desc_iotlb_pasid(uint16_t domain, uint32_t pasid) "iotlb invalidate domain 0x%"PRIx16" pasid 0x%"PRIx32 vtd_inv_desc_wait_sw(uint64_t addr, uint32_t data) "wait invalidate status write addr 0x%"PRIx64" data 0x%"PRIx32 vtd_inv_desc_wait_irq(const char *msg) "%s" vtd_inv_desc_wait_write_fail(uint64_t hi, uint64_t lo) "write fail for wait desc hi 0x%"PRIx64" lo 0x%"PRIx64 diff --git a/include/hw/i386/intel_iommu.h b/include/hw/i386/intel_iommu.h index e49fff2a6c..46d973e629 100644 --- a/include/hw/i386/intel_iommu.h +++ b/include/hw/i386/intel_iommu.h @@ -97,11 +97,13 @@ struct VTDPASIDEntry { struct VTDAddressSpace { PCIBus *bus; uint8_t devfn; + uint32_t pasid; AddressSpace as; IOMMUMemoryRegion iommu; MemoryRegion root; /* The root container of the device */ MemoryRegion nodmar; /* The alias of shared nodmar MR */ MemoryRegion iommu_ir; /* Interrupt region: 0xfeeXXXXX */ + MemoryRegion iommu_ir_fault; /* Interrupt region for catching fault */ IntelIOMMUState *iommu_state; VTDContextCacheEntry context_cache_entry; QLIST_ENTRY(VTDAddressSpace) next; @@ -113,6 +115,7 @@ struct VTDAddressSpace { struct VTDIOTLBEntry { uint64_t gfn; uint16_t domain_id; + uint32_t pasid; uint64_t slpte; uint64_t mask; uint8_t access_flags; @@ -261,6 +264,7 @@ struct IntelIOMMUState { uint8_t aw_bits; /* Host/IOVA address width (in bits) */ bool dma_drain; /* Whether DMA r/w draining enabled */ bool dma_translation; /* Whether DMA translation supported */ + bool pasid; /* Whether to support PASID */ /* * Protects IOMMU states in general. Currently it protects the @@ -272,6 +276,7 @@ struct IntelIOMMUState { /* Find the VTD Address space associated with the given bus pointer, * create a new one if none exists */ -VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn); +VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, + int devfn, unsigned int pasid); #endif diff --git a/include/hw/pci/pci_bus.h b/include/hw/pci/pci_bus.h index eb94e7e85c..5653175957 100644 --- a/include/hw/pci/pci_bus.h +++ b/include/hw/pci/pci_bus.h @@ -28,6 +28,8 @@ enum PCIBusFlags { PCI_BUS_CXL = 0x0004, }; +#define PCI_NO_PASID UINT32_MAX + struct PCIBus { BusState qbus; enum PCIBusFlags flags; -- cgit v1.2.3-55-g7522 From 05e27d74c7dc5318367521f020bf0d4a32228dcc Mon Sep 17 00:00:00 2001 From: Julia Suvorova Date: Tue, 11 Oct 2022 13:17:27 +0200 Subject: hw/smbios: add core_count2 to smbios table type 4 In order to use the increased number of cpus, we need to bring smbios tables in line with the SMBIOS 3.0 specification. This allows us to introduce core_count2 which acts as a duplicate of core_count if we have fewer cores than 256, and contains the actual core number per socket if we have more. core_enabled2 and thread_count2 fields work the same way. Signed-off-by: Julia Suvorova Reviewed-by: Igor Mammedov Message-Id: <20220731162141.178443-2-jusual@redhat.com> Message-Id: <20221011111731.101412-2-jusual@redhat.com> Reviewed-by: Michael S. Tsirkin Signed-off-by: Michael S. Tsirkin --- hw/smbios/smbios.c | 19 ++++++++++++++++--- hw/smbios/smbios_build.h | 9 +++++++-- include/hw/firmware/smbios.h | 12 ++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) (limited to 'include') diff --git a/hw/smbios/smbios.c b/hw/smbios/smbios.c index 51437ca09f..b4243de735 100644 --- a/hw/smbios/smbios.c +++ b/hw/smbios/smbios.c @@ -711,8 +711,14 @@ static void smbios_build_type_3_table(void) static void smbios_build_type_4_table(MachineState *ms, unsigned instance) { char sock_str[128]; + size_t tbl_len = SMBIOS_TYPE_4_LEN_V28; - SMBIOS_BUILD_TABLE_PRE(4, T4_BASE + instance, true); /* required */ + if (smbios_ep_type == SMBIOS_ENTRY_POINT_TYPE_64) { + tbl_len = SMBIOS_TYPE_4_LEN_V30; + } + + SMBIOS_BUILD_TABLE_PRE_SIZE(4, T4_BASE + instance, + true, tbl_len); /* required */ snprintf(sock_str, sizeof(sock_str), "%s%2x", type4.sock_pfx, instance); SMBIOS_TABLE_SET_STR(4, socket_designation_str, sock_str); @@ -739,8 +745,15 @@ static void smbios_build_type_4_table(MachineState *ms, unsigned instance) SMBIOS_TABLE_SET_STR(4, serial_number_str, type4.serial); SMBIOS_TABLE_SET_STR(4, asset_tag_number_str, type4.asset); SMBIOS_TABLE_SET_STR(4, part_number_str, type4.part); - t->core_count = t->core_enabled = ms->smp.cores; - t->thread_count = ms->smp.threads; + + t->core_count = (ms->smp.cores > 255) ? 0xFF : ms->smp.cores; + t->core_enabled = t->core_count; + + t->core_count2 = t->core_enabled2 = cpu_to_le16(ms->smp.cores); + + t->thread_count = (ms->smp.threads > 255) ? 0xFF : ms->smp.threads; + t->thread_count2 = cpu_to_le16(ms->smp.threads); + t->processor_characteristics = cpu_to_le16(0x02); /* Unknown */ t->processor_family2 = cpu_to_le16(0x01); /* Other */ diff --git a/hw/smbios/smbios_build.h b/hw/smbios/smbios_build.h index 56b5a1e3f3..351660024e 100644 --- a/hw/smbios/smbios_build.h +++ b/hw/smbios/smbios_build.h @@ -27,6 +27,11 @@ extern unsigned smbios_table_max; extern unsigned smbios_table_cnt; #define SMBIOS_BUILD_TABLE_PRE(tbl_type, tbl_handle, tbl_required) \ + SMBIOS_BUILD_TABLE_PRE_SIZE(tbl_type, tbl_handle, tbl_required, \ + sizeof(struct smbios_type_##tbl_type))\ + +#define SMBIOS_BUILD_TABLE_PRE_SIZE(tbl_type, tbl_handle, \ + tbl_required, tbl_len) \ struct smbios_type_##tbl_type *t; \ size_t t_off; /* table offset into smbios_tables */ \ int str_index = 0; \ @@ -39,12 +44,12 @@ extern unsigned smbios_table_cnt; /* use offset of table t within smbios_tables */ \ /* (pointer must be updated after each realloc) */ \ t_off = smbios_tables_len; \ - smbios_tables_len += sizeof(*t); \ + smbios_tables_len += tbl_len; \ smbios_tables = g_realloc(smbios_tables, smbios_tables_len); \ t = (struct smbios_type_##tbl_type *)(smbios_tables + t_off); \ \ t->header.type = tbl_type; \ - t->header.length = sizeof(*t); \ + t->header.length = tbl_len; \ t->header.handle = cpu_to_le16(tbl_handle); \ } while (0) diff --git a/include/hw/firmware/smbios.h b/include/hw/firmware/smbios.h index e7d386f7c8..7f3259a630 100644 --- a/include/hw/firmware/smbios.h +++ b/include/hw/firmware/smbios.h @@ -18,6 +18,8 @@ #define SMBIOS_MAX_TYPE 127 +#define offsetofend(TYPE, MEMBER) \ + (offsetof(TYPE, MEMBER) + sizeof_field(TYPE, MEMBER)) /* memory area description, used by type 19 table */ struct smbios_phys_mem_area { @@ -187,8 +189,18 @@ struct smbios_type_4 { uint8_t thread_count; uint16_t processor_characteristics; uint16_t processor_family2; + /* SMBIOS spec 3.0.0, Table 21 */ + uint16_t core_count2; + uint16_t core_enabled2; + uint16_t thread_count2; } QEMU_PACKED; +typedef enum smbios_type_4_len_ver { + SMBIOS_TYPE_4_LEN_V28 = offsetofend(struct smbios_type_4, + processor_family2), + SMBIOS_TYPE_4_LEN_V30 = offsetofend(struct smbios_type_4, thread_count2), +} smbios_type_4_len_ver; + /* SMBIOS type 8 - Port Connector Information */ struct smbios_type_8 { struct smbios_structure_header header; -- cgit v1.2.3-55-g7522 From 259d69c00b67c02a67f3bdbeeea71c2c0af76c35 Mon Sep 17 00:00:00 2001 From: Alex Bennée Date: Mon, 7 Nov 2022 12:14:07 +0000 Subject: hw/virtio: introduce virtio_device_should_start The previous fix to virtio_device_started revealed a problem in its use by both the core and the device code. The core code should be able to handle the device "starting" while the VM isn't running to handle the restoration of migration state. To solve this duel use introduce a new helper for use by the vhost-user backends who all use it to feed a should_start variable. We can also pick up a change vhost_user_blk_set_status while we are at it which follows the same pattern. Fixes: 9f6bcfd99f (hw/virtio: move vm_running check to virtio_device_started) Signed-off-by: Alex Bennée Cc: "Michael S. Tsirkin" Message-Id: <20221107121407.1010913-1-alex.bennee@linaro.org> --- hw/block/vhost-user-blk.c | 6 +----- hw/virtio/vhost-user-fs.c | 2 +- hw/virtio/vhost-user-gpio.c | 2 +- hw/virtio/vhost-user-i2c.c | 2 +- hw/virtio/vhost-user-rng.c | 2 +- hw/virtio/vhost-user-vsock.c | 2 +- hw/virtio/vhost-vsock.c | 2 +- include/hw/virtio/virtio.h | 18 ++++++++++++++++++ 8 files changed, 25 insertions(+), 11 deletions(-) (limited to 'include') diff --git a/hw/block/vhost-user-blk.c b/hw/block/vhost-user-blk.c index 28409c90f7..16ad400889 100644 --- a/hw/block/vhost-user-blk.c +++ b/hw/block/vhost-user-blk.c @@ -226,14 +226,10 @@ static void vhost_user_blk_stop(VirtIODevice *vdev) static void vhost_user_blk_set_status(VirtIODevice *vdev, uint8_t status) { VHostUserBlk *s = VHOST_USER_BLK(vdev); - bool should_start = virtio_device_started(vdev, status); + bool should_start = virtio_device_should_start(vdev, status); Error *local_err = NULL; int ret; - if (!vdev->vm_running) { - should_start = false; - } - if (!s->connected) { return; } diff --git a/hw/virtio/vhost-user-fs.c b/hw/virtio/vhost-user-fs.c index ad0f91c607..1c40f42045 100644 --- a/hw/virtio/vhost-user-fs.c +++ b/hw/virtio/vhost-user-fs.c @@ -123,7 +123,7 @@ static void vuf_stop(VirtIODevice *vdev) static void vuf_set_status(VirtIODevice *vdev, uint8_t status) { VHostUserFS *fs = VHOST_USER_FS(vdev); - bool should_start = virtio_device_started(vdev, status); + bool should_start = virtio_device_should_start(vdev, status); if (vhost_dev_is_started(&fs->vhost_dev) == should_start) { return; diff --git a/hw/virtio/vhost-user-gpio.c b/hw/virtio/vhost-user-gpio.c index 8b40fe450c..677d1c7730 100644 --- a/hw/virtio/vhost-user-gpio.c +++ b/hw/virtio/vhost-user-gpio.c @@ -152,7 +152,7 @@ static void vu_gpio_stop(VirtIODevice *vdev) static void vu_gpio_set_status(VirtIODevice *vdev, uint8_t status) { VHostUserGPIO *gpio = VHOST_USER_GPIO(vdev); - bool should_start = virtio_device_started(vdev, status); + bool should_start = virtio_device_should_start(vdev, status); trace_virtio_gpio_set_status(status); diff --git a/hw/virtio/vhost-user-i2c.c b/hw/virtio/vhost-user-i2c.c index bc58b6c0d1..864eba695e 100644 --- a/hw/virtio/vhost-user-i2c.c +++ b/hw/virtio/vhost-user-i2c.c @@ -93,7 +93,7 @@ static void vu_i2c_stop(VirtIODevice *vdev) static void vu_i2c_set_status(VirtIODevice *vdev, uint8_t status) { VHostUserI2C *i2c = VHOST_USER_I2C(vdev); - bool should_start = virtio_device_started(vdev, status); + bool should_start = virtio_device_should_start(vdev, status); if (vhost_dev_is_started(&i2c->vhost_dev) == should_start) { return; diff --git a/hw/virtio/vhost-user-rng.c b/hw/virtio/vhost-user-rng.c index bc1f36c5ac..8b47287875 100644 --- a/hw/virtio/vhost-user-rng.c +++ b/hw/virtio/vhost-user-rng.c @@ -90,7 +90,7 @@ static void vu_rng_stop(VirtIODevice *vdev) static void vu_rng_set_status(VirtIODevice *vdev, uint8_t status) { VHostUserRNG *rng = VHOST_USER_RNG(vdev); - bool should_start = virtio_device_started(vdev, status); + bool should_start = virtio_device_should_start(vdev, status); if (vhost_dev_is_started(&rng->vhost_dev) == should_start) { return; diff --git a/hw/virtio/vhost-user-vsock.c b/hw/virtio/vhost-user-vsock.c index 7b67e29d83..9431b9792c 100644 --- a/hw/virtio/vhost-user-vsock.c +++ b/hw/virtio/vhost-user-vsock.c @@ -55,7 +55,7 @@ const VhostDevConfigOps vsock_ops = { static void vuv_set_status(VirtIODevice *vdev, uint8_t status) { VHostVSockCommon *vvc = VHOST_VSOCK_COMMON(vdev); - bool should_start = virtio_device_started(vdev, status); + bool should_start = virtio_device_should_start(vdev, status); if (vhost_dev_is_started(&vvc->vhost_dev) == should_start) { return; diff --git a/hw/virtio/vhost-vsock.c b/hw/virtio/vhost-vsock.c index 7dc3c73931..aa16d584ee 100644 --- a/hw/virtio/vhost-vsock.c +++ b/hw/virtio/vhost-vsock.c @@ -70,7 +70,7 @@ static int vhost_vsock_set_running(VirtIODevice *vdev, int start) static void vhost_vsock_set_status(VirtIODevice *vdev, uint8_t status) { VHostVSockCommon *vvc = VHOST_VSOCK_COMMON(vdev); - bool should_start = virtio_device_started(vdev, status); + bool should_start = virtio_device_should_start(vdev, status); int ret; if (vhost_dev_is_started(&vvc->vhost_dev) == should_start) { diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h index 1423dba379..141a253a2c 100644 --- a/include/hw/virtio/virtio.h +++ b/include/hw/virtio/virtio.h @@ -395,6 +395,24 @@ static inline bool virtio_device_started(VirtIODevice *vdev, uint8_t status) return vdev->started; } + return status & VIRTIO_CONFIG_S_DRIVER_OK; +} + +/** + * virtio_device_should_start() - check if device startable + * @vdev - the VirtIO device + * @status - the devices status bits + * + * This is similar to virtio_device_started() but also encapsulates a + * check on the VM status which would prevent a device starting + * anyway. + */ +static inline bool virtio_device_should_start(VirtIODevice *vdev, uint8_t status) +{ + if (vdev->use_started) { + return vdev->started; + } + if (!vdev->vm_running) { return false; } -- cgit v1.2.3-55-g7522