path: root/src/net/infiniband.c

/*
 * Copyright (C) 2007 Michael Brown <mbrown@fensystems.co.uk>.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

#include <stdint.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <byteswap.h>
#include <errno.h>
#include <assert.h>
#include <gpxe/list.h>
#include <gpxe/if_arp.h>
#include <gpxe/netdevice.h>
#include <gpxe/iobuf.h>
#include <gpxe/infiniband.h>

/** @file
 *
 * Infiniband protocol
 *
 */

/**
 * Create completion queue
 *
 * @v ibdev		Infiniband device
 * @v num_cqes		Number of completion queue entries
 * @ret cq		New completion queue
 */
struct ib_completion_queue * ib_create_cq ( struct ib_device *ibdev,
					    unsigned int num_cqes ) {
	struct ib_completion_queue *cq;
	int rc;

	DBGC ( ibdev, "IBDEV %p creating completion queue\n", ibdev );

	/* Allocate and initialise data structure */
	cq = zalloc ( sizeof ( *cq ) );
	if ( ! cq )
		return NULL;
	cq->num_cqes = num_cqes;
	INIT_LIST_HEAD ( &cq->work_queues );

	/* Perform device-specific initialisation and get CQN */
	if ( ( rc = ibdev->op->create_cq ( ibdev, cq ) ) != 0 ) {
		DBGC ( ibdev, "IBDEV %p could not initialise completion "
		       "queue: %s\n", ibdev, strerror ( rc ) );
		free ( cq );
		return NULL;
	}

	DBGC ( ibdev, "IBDEV %p created completion queue %#lx\n",
	       ibdev, cq->cqn );
	return cq;
}

/**
 * Destroy completion queue
 *
 * @v ibdev		Infiniband device
 * @v cq		Completion queue
 */
void ib_destroy_cq ( struct ib_device *ibdev,
		     struct ib_completion_queue *cq ) {
	DBGC ( ibdev, "IBDEV %p destroying completion queue %#lx\n",
	       ibdev, cq->cqn );
	assert ( list_empty ( &cq->work_queues ) );
	ibdev->op->destroy_cq ( ibdev, cq );
	free ( cq );
}

/**
 * Create queue pair
 *
 * @v ibdev		Infiniband device
 * @v num_send_wqes	Number of send work queue entries
 * @v send_cq		Send completion queue
 * @v num_recv_wqes	Number of receive work queue entries
 * @v recv_cq		Receive completion queue
 * @ret qp		Queue pair
 */
struct ib_queue_pair * ib_create_qp ( struct ib_device *ibdev,
				      unsigned int num_send_wqes,
				      struct ib_completion_queue *send_cq,
				      unsigned int num_recv_wqes,
				      struct ib_completion_queue *recv_cq ) {
	struct ib_queue_pair *qp;
	int rc;

	DBGC ( ibdev, "IBDEV %p creating queue pair\n", ibdev );

	/* Allocate and initialise data structure */
	qp = zalloc ( sizeof ( *qp ) +
		      ( num_send_wqes * sizeof ( qp->send.iobufs[0] ) ) +
		      ( num_recv_wqes * sizeof ( qp->recv.iobufs[0] ) ) );
	if ( ! qp )
		return NULL;
	qp->send.qp = qp;
	qp->send.is_send = 1;
	qp->send.cq = send_cq;
	list_add ( &qp->send.list, &send_cq->work_queues );
	qp->send.num_wqes = num_send_wqes;
	qp->send.iobufs = ( ( ( void * ) qp ) + sizeof ( *qp ) );
	qp->recv.qp = qp;
	qp->recv.cq = recv_cq;
	list_add ( &qp->recv.list, &recv_cq->work_queues );
	qp->recv.num_wqes = num_recv_wqes;
	qp->recv.iobufs = ( ( ( void * ) qp ) + sizeof ( *qp ) +
			    ( num_send_wqes * sizeof ( qp->send.iobufs[0] ) ));

	/* Perform device-specific initialisation and get QPN */
	if ( ( rc = ibdev->op->create_qp ( ibdev, qp ) ) != 0 ) {
		DBGC ( ibdev, "IBDEV %p could not initialise queue pair: "
		       "%s\n", ibdev, strerror ( rc ) );
		free ( qp );
		return NULL;
	}

	DBGC ( ibdev, "IBDEV %p created queue pair %#lx\n",
	       ibdev, qp->qpn );
	return qp;
}

/**
 * Destroy queue pair
 *
 * @v ibdev		Infiniband device
 * @v qp		Queue pair
 */
void ib_destroy_qp ( struct ib_device *ibdev,
		     struct ib_queue_pair *qp ) {
	DBGC ( ibdev, "IBDEV %p destroying queue pair %#lx\n",
	       ibdev, qp->qpn );
	ibdev->op->destroy_qp ( ibdev, qp );
	free ( qp );
}



/**
 * Find work queue belonging to completion queue
 *
 * @v cq		Completion queue
 * @v qpn		Queue pair number
 * @v is_send		Find send work queue (rather than receive)
 * @ret wq		Work queue, or NULL if not found
 */
struct ib_work_queue * ib_find_wq ( struct ib_completion_queue *cq,
				    unsigned long qpn, int is_send ) {
	struct ib_work_queue *wq;

	list_for_each_entry ( wq, &cq->work_queues, list ) {
		if ( ( wq->qp->qpn == qpn ) && ( wq->is_send == is_send ) )
			return wq;
	}
	return NULL;
}



/** Infiniband broadcast MAC address */
static uint8_t ib_broadcast[IB_ALEN] = { 0xff, };

/**
 * Transmit Infiniband packet
 *
 * @v iobuf		I/O buffer
 * @v netdev		Network device
 * @v net_protocol	Network-layer protocol
 * @v ll_dest		Link-layer destination address
 *
 * Prepends the Infiniband link-layer header and transmits the packet.
 */
static int ib_tx ( struct io_buffer *iobuf, struct net_device *netdev,
		   struct net_protocol *net_protocol, const void *ll_dest ) {
	struct ibhdr *ibhdr = iob_push ( iobuf, sizeof ( *ibhdr ) );

	/* Build Infiniband header */
	ibhdr->proto = net_protocol->net_proto;
	ibhdr->reserved = 0;

	( void ) ll_dest;

	/* Hand off to network device */
	return netdev_tx ( netdev, iobuf );
}

/**
 * Process received Infiniband packet
 *
 * @v iobuf	I/O buffer
 * @v netdev	Network device
 *
 * Strips off the Infiniband link-layer header and passes up to the
 * network-layer protocol.
 */
static int ib_rx ( struct io_buffer *iobuf, struct net_device *netdev ) {
	struct ibhdr *ibhdr = iobuf->data;

	/* Sanity check */
	if ( iob_len ( iobuf ) < sizeof ( *ibhdr ) ) {
		DBG ( "Infiniband packet too short (%d bytes)\n",
		      iob_len ( iobuf ) );
		free_iob ( iobuf );
		return -EINVAL;
	}

	/* Strip off Infiniband header */
	iob_pull ( iobuf, sizeof ( *ibhdr ) );

	/* Hand off to network-layer protocol */
	return net_rx ( iobuf, netdev, ibhdr->proto, NULL );
}

/**
 * Transcribe Infiniband address
 *
 * @v ll_addr	Link-layer address
 * @ret string	Link-layer address in human-readable format
 */
const char * ib_ntoa ( const void *ll_addr ) {
	static char buf[61];
	const uint8_t *ib_addr = ll_addr;
	unsigned int i;
	char *p = buf;

	for ( i = 0 ; i < IB_ALEN ; i++ ) {
		p += sprintf ( p, ":%02x", ib_addr[i] );
	}
	return ( buf + 1 );
}

/** Infiniband protocol */
struct ll_protocol infiniband_protocol __ll_protocol = {
	.name		= "Infiniband",
	.ll_proto	= htons ( ARPHRD_INFINIBAND ),
	.ll_addr_len	= IB_ALEN,
	.ll_header_len	= IB_HLEN,
	.ll_broadcast	= ib_broadcast,
	.tx		= ib_tx,
	.rx		= ib_rx,
	.ntoa		= ib_ntoa,
};