/*
* Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/mlx5/driver.h>
#include "mlx5_core.h"
#include <linux/mlx5/transobj.h>
int mlx5_core_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn)
{
u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {0};
u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {0};
int err;
MLX5_SET(alloc_transport_domain_in, in, opcode,
MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);
err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
if (!err)
*tdn = MLX5_GET(alloc_transport_domain_out, out,
transport_domain);
return err;
}
EXPORT_SYMBOL(mlx5_core_alloc_transport_domain);
void mlx5_core_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn)
{
u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {0};
u32 out[MLX5_ST_SZ_DW(dealloc_transport_domain_out)] = {0};
MLX5_SET(dealloc_transport_domain_in, in, opcode,
MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);
mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
EXPORT_SYMBOL(mlx5_core_dealloc_transport_domain);
int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rqn)
{
u32 out[MLX5_ST_SZ_DW(create_rq_out)] = {0};
int err;
MLX5_SET(create_rq_in, in, opcode, MLX5_CMD_OP_CREATE_RQ);
err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
if (!err)
*rqn = MLX5_GET(create_rq_out, out, rqn);
return err;
}
EXPORT_SYMBOL(mlx5_core_create_rq);
int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *in, int inlen)
{
u32 out[MLX5_ST_SZ_DW(modify_rq_out)];
MLX5_SET(modify_rq_in, in, rqn, rqn);
MLX5_SET(modify_rq_in, in, opcode, MLX5_CMD_OP_MODIFY_RQ);
memset(out, 0, sizeof(out));
return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
}
EXPORT_SYMBOL(mlx5_core_modify_rq);
void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn)
{
u32 in[MLX5_ST_SZ_DW(destroy_rq_in)] = {0};
u32 out[MLX5_ST_SZ_DW(destroy_rq_out)] = {0};
MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ);
MLX5_SET(destroy_rq_in, in, rqn, rqn);
mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
EXPORT_SYMBOL(mlx5_core_destroy_rq);
int mlx5_core_query_rq(struct mlx5_core_dev *dev, u32 rqn, u32 *out)
{
u32 in[MLX5_ST_SZ_DW(query_rq_in)] = {0};
int outlen = MLX5_ST_SZ_BYTES(query_rq_out);
MLX5_SET(query_rq_in, in, opcode, MLX5_CMD_OP_QUERY_RQ);
MLX5_SET(query_rq_in, in, rqn, rqn);
return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
}
EXPORT_SYMBOL(mlx5_core_query_rq);
int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *sqn)
{
u32 out[MLX5_ST_SZ_DW(create_sq_out)] = {0};
int err;
MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);
err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
if (!err)
*sqn = MLX5_GET(create_sq_out, out, sqn);
return err;
}
int mlx5_core_modify_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *in, int inlen)
{
u32 out[MLX5_ST_SZ_DW(modify_sq_out)] = {0};
MLX5_SET(modify_sq_in, in, sqn, sqn);
MLX5_SET(modify_sq_in, in, opcode, MLX5_CMD_OP_MODIFY_SQ);
return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
}
EXPORT_SYMBOL(mlx5_core_modify_sq);
void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn)
{
u32 in[MLX5_ST_SZ_DW(destroy_sq_in)] = {0};
u32 out[MLX5_ST_SZ_DW(destroy_sq_out)] = {0};
MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ);
MLX5_SET(destroy_sq_in, in, sqn, sqn);
mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
int mlx5_core_query_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *out)
{
u32 in[MLX5_ST_SZ_DW(query_sq_in)] = {0};
int outlen = MLX5_ST_SZ_BYTES(query_sq_out);
MLX5_SET(query_sq_in, in, opcode, MLX5_CMD_OP_QUERY_SQ);
MLX5_SET(query_sq_in, in, sqn, sqn);
return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
}
EXPORT_SYMBOL(mlx5_core_query_sq);
int mlx5_core_query_sq_state(struct mlx5_core_dev *dev, u32 sqn, u8 *state)
{
void *out;
void *sqc;
int inlen;
int err;
inlen = MLX5_ST_SZ_BYTES(query_sq_out);
out = kvzalloc(inlen, GFP_KERNEL);
if (!out)
return -ENOMEM;
err = mlx5_core_query_sq(dev, sqn, out);
if (err)
goto out;
sqc = MLX5_ADDR_OF(query_sq_out, out, sq_context);
*state = MLX5_GET(sqc, sqc, state);
out:
kvfree(out);
return err;
}
EXPORT_SYMBOL_GPL(mlx5_core_query_sq_state);
int mlx5_core_create_tir_out(struct mlx5_core_dev *dev,
u32 *in, int inlen,
u32 *out, int outlen)
{
MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
return mlx5_cmd_exec(dev, in, inlen, out, outlen);
}
EXPORT_SYMBOL(mlx5_core_create_tir_out);
int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, int inlen,
u32 *tirn)
{
u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
int err;
err = mlx5_core_create_tir_out(dev, in, inlen,
out, sizeof(out));
if (!err)
*tirn = MLX5_GET(create_tir_out, out, tirn);
return err;
}
EXPORT_SYMBOL(mlx5_core_create_tir);
int mlx5_core_modify_tir(struct mlx5_core_dev *dev, u32 tirn, u32 *in,
int inlen)
{
u32 out[MLX5_ST_SZ_DW(modify_tir_out)] = {0};
MLX5_SET(modify_tir_in, in, tirn, tirn);
MLX5_SET(modify_tir_in, in, opcode, MLX5_CMD_OP_MODIFY_TIR);
return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
}
void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn)
{
u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {0};
u32 out[MLX5_ST_SZ_DW(destroy_tir_out)] = {0};
MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);
MLX5_SET(destroy_tir_in, in, tirn, tirn);
mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
EXPORT_SYMBOL(mlx5_core_destroy_tir);
int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, int inlen,
u32 *tisn)
{
u32 out[MLX5_ST_SZ_DW(create_tis_out)] = {0};
int err;
MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);
err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
if (!err)
*tisn = MLX5_GET(create_tis_out, out, tisn);
return err;
}
EXPORT_SYMBOL(mlx5_core_create_tis);
int mlx5_core_modify_tis(struct mlx5_core_dev *dev, u32 tisn, u32 *in,
int inlen)
{
u32 out[MLX5_ST_SZ_DW(modify_tis_out)] = {0};
MLX5_SET(modify_tis_in, in, tisn, tisn);
MLX5_SET(modify_tis_in, in, opcode, MLX5_CMD_OP_MODIFY_TIS);
return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
}
EXPORT_SYMBOL(mlx5_core_modify_tis);
void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn)
{
u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {0};
u32 out[MLX5_ST_SZ_DW(destroy_tis_out)] = {0};
MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);
MLX5_SET(destroy_tis_in, in, tisn, tisn);
mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
EXPORT_SYMBOL(mlx5_core_destroy_tis);
int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen,
u32 *rqtn)
{
u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {0};
int err;
MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
if (!err)
*rqtn = MLX5_GET(create_rqt_out, out, rqtn);
return err;
}
EXPORT_SYMBOL(mlx5_core_create_rqt);
int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in,
int inlen)
{
u32 out[MLX5_ST_SZ_DW(modify_rqt_out)] = {0};
MLX5_SET(modify_rqt_in, in, rqtn, rqtn);
MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT);
return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
}
void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn)
{
u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {0};
u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)] = {0};
MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);
mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
EXPORT_SYMBOL(mlx5_core_destroy_rqt);
static int mlx5_hairpin_create_rq(struct mlx5_core_dev *mdev,
struct mlx5_hairpin_params *params, u32 *rqn)
{
u32 in[MLX5_ST_SZ_DW(create_rq_in)] = {0};
void *rqc, *wq;
rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
wq = MLX5_ADDR_OF(rqc, rqc, wq);
MLX5_SET(rqc, rqc, hairpin, 1);
MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
MLX5_SET(rqc, rqc, counter_set_id, params->q_counter);
MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);
return mlx5_core_create_rq(mdev, in, MLX5_ST_SZ_BYTES(create_rq_in), rqn);
}
static int mlx5_hairpin_create_sq(struct mlx5_core_dev *mdev,
struct mlx5_hairpin_params *params, u32 *sqn)
{
u32 in[MLX5_ST_SZ_DW(create_sq_in)] = {0};
void *sqc, *wq;
sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
wq = MLX5_ADDR_OF(sqc, sqc, wq);
MLX5_SET(sqc, sqc, hairpin, 1);
MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
MLX5_SET(wq, wq, log_hairpin_data_sz, params->log_data_size);
MLX5_SET(wq, wq, log_hairpin_num_packets, params->log_num_packets);
return mlx5_core_create_sq(mdev, in, MLX5_ST_SZ_BYTES(create_sq_in), sqn);
}
static int mlx5_hairpin_create_queues(struct mlx5_hairpin *hp,
struct mlx5_hairpin_params *params)
{
int i, j, err;
for (i = 0; i < hp->num_channels; i++) {
err = mlx5_hairpin_create_rq(hp->func_mdev, params, &hp->rqn[i]);
if (err)
goto out_err_rq;
}
for (i = 0; i < hp->num_channels; i++) {
err = mlx5_hairpin_create_sq(hp->peer_mdev, params, &hp->sqn[i]);
if (err)
goto out_err_sq;
}
return 0;
out_err_sq:
for (j = 0; j < i; j++)
mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[j]);
i = hp->num_channels;
out_err_rq:
for (j = 0; j < i; j++)
mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[j]);
return err;
}
static void mlx5_hairpin_destroy_queues(struct mlx5_hairpin *hp)
{
int i;
for (i = 0; i < hp->num_channels; i++) {
mlx5_core_destroy_rq(hp->func_mdev, hp->rqn[i]);
if (!hp->peer_gone)
mlx5_core_destroy_sq(hp->peer_mdev, hp->sqn[i]);
}
}
static int mlx5_hairpin_modify_rq(struct mlx5_core_dev *func_mdev, u32 rqn,
int curr_state, int next_state,
u16 peer_vhca, u32 peer_sq)
{
u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {0};
void *rqc;
rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
if (next_state == MLX5_RQC_STATE_RDY) {
MLX5_SET(rqc, rqc, hairpin_peer_sq, peer_sq);
MLX5_SET(rqc, rqc, hairpin_peer_vhca, peer_vhca);
}
MLX5_SET(modify_rq_in, in, rq_state, curr_state);
MLX5_SET(rqc, rqc, state, next_state);
return mlx5_core_modify_rq(func_mdev, rqn,
in, MLX5_ST_SZ_BYTES(modify_rq_in));
}
static int mlx5_hairpin_modify_sq(struct mlx5_core_dev *peer_mdev, u32 sqn,
int curr_state, int next_state,
u16 peer_vhca, u32 peer_rq)
{
u32 in[MLX5_ST_SZ_DW(modify_sq_in)] = {0};
void *sqc;
sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
if (next_state == MLX5_SQC_STATE_RDY) {
MLX5_SET(sqc, sqc, hairpin_peer_rq, peer_rq);
MLX5_SET(sqc, sqc, hairpin_peer_vhca, peer_vhca);
}
MLX5_SET(modify_sq_in, in, sq_state, curr_state);
MLX5_SET(sqc, sqc, state, next_state);
return mlx5_core_modify_sq(peer_mdev, sqn,
in, MLX5_ST_SZ_BYTES(modify_sq_in));
}
static int mlx5_hairpin_pair_queues(struct mlx5_hairpin *hp)
{
int i, j, err;
/* set peer SQs */
for (i = 0; i < hp->num_channels; i++) {
err = mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i],
MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY,
MLX5_CAP_GEN(hp->func_mdev, vhca_id), hp->rqn[i]);
if (err)
goto err_modify_sq;
}
/* set func RQs */
for (i = 0; i < hp->num_channels; i++) {
err = mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i],
MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY,
MLX5_CAP_GEN(hp->peer_mdev, vhca_id), hp->sqn[i]);
if (err)
goto err_modify_rq;
}
return 0;
err_modify_rq:
for (j = 0; j < i; j++)
mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[j], MLX5_RQC_STATE_RDY,
MLX5_RQC_STATE_RST, 0, 0);
i = hp->num_channels;
err_modify_sq:
for (j = 0; j < i; j++)
mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[j], MLX5_SQC_STATE_RDY,
MLX5_SQC_STATE_RST, 0, 0);
return err;
}
static void mlx5_hairpin_unpair_queues(struct mlx5_hairpin *hp)
{
int i;
/* unset func RQs */
for (i = 0; i < hp->num_channels; i++)
mlx5_hairpin_modify_rq(hp->func_mdev, hp->rqn[i], MLX5_RQC_STATE_RDY,
MLX5_RQC_STATE_RST, 0, 0);
/* unset peer SQs */
if (hp->peer_gone)
return;
for (i = 0; i < hp->num_channels; i++)
mlx5_hairpin_modify_sq(hp->peer_mdev, hp->sqn[i], MLX5_SQC_STATE_RDY,
MLX5_SQC_STATE_RST, 0, 0);
}
struct mlx5_hairpin *
mlx5_core_hairpin_create(struct mlx5_core_dev *func_mdev,
struct mlx5_core_dev *peer_mdev,
struct mlx5_hairpin_params *params)
{
struct mlx5_hairpin *hp;
int size, err;
size = sizeof(*hp) + params->num_channels * 2 * sizeof(u32);
hp = kzalloc(size, GFP_KERNEL);
if (!hp)
return ERR_PTR(-ENOMEM);
hp->func_mdev = func_mdev;
hp->peer_mdev = peer_mdev;
hp->num_channels = params->num_channels;
hp->rqn = (void *)hp + sizeof(*hp);
hp->sqn = hp->rqn + params->num_channels;
/* alloc and pair func --> peer hairpin */
err = mlx5_hairpin_create_queues(hp, params);
if (err)
goto err_create_queues;
err = mlx5_hairpin_pair_queues(hp);
if (err)
goto err_pair_queues;
return hp;
err_pair_queues:
mlx5_hairpin_destroy_queues(hp);
err_create_queues:
kfree(hp);
return ERR_PTR(err);
}
void mlx5_core_hairpin_destroy(struct mlx5_hairpin *hp)
{
mlx5_hairpin_unpair_queues(hp);
mlx5_hairpin_destroy_queues(hp);
kfree(hp);
}
