From: Roland Dreier <roland@topspin.com>
To: akpm@osdl.org
Cc: linux-kernel@vger.kernel.org, openib-general@openib.org
Subject: [PATCH][3/18] InfiniBand/mthca: support RDMA/atomic attributes in QP modify
Date: Wed, 12 Jan 2005 13:47:46 -0800 [thread overview]
Message-ID: <20051121347.gzXWVDbfUCDwWmdf@topspin.com> (raw)
In-Reply-To: <20051121347.SllTsxtwWAwPk3Gh@topspin.com>
Implement setting of RDMA/atomic enable bits, initiator resources and
responder resources for modify QP in low-level Mellanox HCA driver
(should complete RDMA/atomic implementation).
Signed-off-by: Roland Dreier <roland@topspin.com>
--- linux/drivers/infiniband/hw/mthca/mthca_dev.h (revision 1421)
+++ linux/drivers/infiniband/hw/mthca/mthca_dev.h (revision 1422)
@@ -75,6 +75,7 @@
MTHCA_EQ_CONTEXT_SIZE = 0x40,
MTHCA_CQ_CONTEXT_SIZE = 0x40,
MTHCA_QP_CONTEXT_SIZE = 0x200,
+ MTHCA_RDB_ENTRY_SIZE = 0x20,
MTHCA_AV_SIZE = 0x20,
MTHCA_MGM_ENTRY_SIZE = 0x40
};
@@ -121,7 +122,6 @@
int mtt_seg_size;
int reserved_mtts;
int reserved_mrws;
- int num_rdbs;
int reserved_uars;
int num_mgms;
int num_amgms;
@@ -174,6 +174,8 @@
struct mthca_qp_table {
struct mthca_alloc alloc;
+ u32 rdb_base;
+ int rdb_shift;
int sqp_start;
spinlock_t lock;
struct mthca_array qp;
--- linux/drivers/infiniband/hw/mthca/mthca_provider.h (revision 1421)
+++ linux/drivers/infiniband/hw/mthca/mthca_provider.h (revision 1422)
@@ -162,9 +162,12 @@
spinlock_t lock;
atomic_t refcount;
u32 qpn;
- int transport;
- enum ib_qp_state state;
int is_direct;
+ u8 transport;
+ u8 state;
+ u8 atomic_rd_en;
+ u8 resp_depth;
+
struct mthca_mr mr;
struct mthca_wq rq;
--- linux/drivers/infiniband/hw/mthca/mthca_profile.c (revision 1421)
+++ linux/drivers/infiniband/hw/mthca/mthca_profile.c (revision 1422)
@@ -50,7 +50,6 @@
};
enum {
- MTHCA_RDB_ENTRY_SIZE = 32,
MTHCA_MTT_SEG_SIZE = 64
};
@@ -181,8 +180,13 @@
init_hca->log_num_eqs = profile[i].log_num;
break;
case MTHCA_RES_RDB:
- dev->limits.num_rdbs = profile[i].num;
- init_hca->rdb_base = profile[i].start;
+ for (dev->qp_table.rdb_shift = 0;
+ profile[MTHCA_RES_QP].num << dev->qp_table.rdb_shift <
+ profile[i].num;
+ ++dev->qp_table.rdb_shift)
+ ; /* nothing */
+ dev->qp_table.rdb_base = (u32) profile[i].start;
+ init_hca->rdb_base = profile[i].start;
break;
case MTHCA_RES_MCG:
dev->limits.num_mgms = profile[i].num >> 1;
--- linux/drivers/infiniband/hw/mthca/mthca_qp.c (revision 1421)
+++ linux/drivers/infiniband/hw/mthca/mthca_qp.c (revision 1422)
@@ -146,7 +146,7 @@
MTHCA_QP_OPTPAR_ALT_ADDR_PATH = 1 << 0,
MTHCA_QP_OPTPAR_RRE = 1 << 1,
MTHCA_QP_OPTPAR_RAE = 1 << 2,
- MTHCA_QP_OPTPAR_REW = 1 << 3,
+ MTHCA_QP_OPTPAR_RWE = 1 << 3,
MTHCA_QP_OPTPAR_PKEY_INDEX = 1 << 4,
MTHCA_QP_OPTPAR_Q_KEY = 1 << 5,
MTHCA_QP_OPTPAR_RNR_TIMEOUT = 1 << 6,
@@ -697,14 +697,87 @@
qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RETRY_COUNT);
}
- /* XXX initiator resources */
+ if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
+ qp_context->params1 |= cpu_to_be32(min(attr->max_dest_rd_atomic ?
+ ffs(attr->max_dest_rd_atomic) - 1 : 0,
+ 7) << 21);
+ qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_SRA_MAX);
+ }
if (attr_mask & IB_QP_SQ_PSN)
qp_context->next_send_psn = cpu_to_be32(attr->sq_psn);
qp_context->cqn_snd = cpu_to_be32(to_mcq(ibqp->send_cq)->cqn);
- /* XXX RDMA/atomic enable, responder resources */
+ if (attr_mask & IB_QP_ACCESS_FLAGS) {
+ /*
+ * Only enable RDMA/atomics if we have responder
+ * resources set to a non-zero value.
+ */
+ if (qp->resp_depth) {
+ qp_context->params2 |=
+ cpu_to_be32(attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE ?
+ MTHCA_QP_BIT_RWE : 0);
+ qp_context->params2 |=
+ cpu_to_be32(attr->qp_access_flags & IB_ACCESS_REMOTE_READ ?
+ MTHCA_QP_BIT_RRE : 0);
+ qp_context->params2 |=
+ cpu_to_be32(attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC ?
+ MTHCA_QP_BIT_RAE : 0);
+ }
+ qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RWE |
+ MTHCA_QP_OPTPAR_RRE |
+ MTHCA_QP_OPTPAR_RAE);
+
+ qp->atomic_rd_en = attr->qp_access_flags;
+ }
+
+ if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
+ u8 rra_max;
+
+ if (qp->resp_depth && !attr->max_rd_atomic) {
+ /*
+ * Lowering our responder resources to zero.
+ * Turn off RDMA/atomics as responder.
+ * (RWE/RRE/RAE in params2 already zero)
+ */
+ qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RWE |
+ MTHCA_QP_OPTPAR_RRE |
+ MTHCA_QP_OPTPAR_RAE);
+ }
+
+ if (!qp->resp_depth && attr->max_rd_atomic) {
+ /*
+ * Increasing our responder resources from
+ * zero. Turn on RDMA/atomics as appropriate.
+ */
+ qp_context->params2 |=
+ cpu_to_be32(qp->atomic_rd_en & IB_ACCESS_REMOTE_WRITE ?
+ MTHCA_QP_BIT_RWE : 0);
+ qp_context->params2 |=
+ cpu_to_be32(qp->atomic_rd_en & IB_ACCESS_REMOTE_READ ?
+ MTHCA_QP_BIT_RRE : 0);
+ qp_context->params2 |=
+ cpu_to_be32(qp->atomic_rd_en & IB_ACCESS_REMOTE_ATOMIC ?
+ MTHCA_QP_BIT_RAE : 0);
+
+ qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RWE |
+ MTHCA_QP_OPTPAR_RRE |
+ MTHCA_QP_OPTPAR_RAE);
+ }
+
+ for (rra_max = 0;
+ 1 << rra_max < attr->max_rd_atomic &&
+ rra_max < dev->qp_table.rdb_shift;
+ ++rra_max)
+ ; /* nothing */
+
+ qp_context->params2 |= cpu_to_be32(rra_max << 21);
+ qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RRA_MAX);
+
+ qp->resp_depth = attr->max_rd_atomic;
+ }
+
if (qp->rq.policy == IB_SIGNAL_ALL_WR)
qp_context->params2 |= cpu_to_be32(MTHCA_QP_BIT_RSC);
if (attr_mask & IB_QP_MIN_RNR_TIMER) {
@@ -714,7 +787,9 @@
if (attr_mask & IB_QP_RQ_PSN)
qp_context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
- /* XXX ra_buff_indx */
+ qp_context->ra_buff_indx = dev->qp_table.rdb_base +
+ ((qp->qpn & (dev->limits.num_qps - 1)) * MTHCA_RDB_ENTRY_SIZE <<
+ dev->qp_table.rdb_shift);
qp_context->cqn_rcv = cpu_to_be32(to_mcq(ibqp->recv_cq)->cqn);
@@ -910,6 +985,8 @@
spin_lock_init(&qp->lock);
atomic_set(&qp->refcount, 1);
qp->state = IB_QPS_RESET;
+ qp->atomic_rd_en = 0;
+ qp->resp_depth = 0;
qp->sq.policy = send_policy;
qp->rq.policy = recv_policy;
qp->rq.cur = 0;
next prev parent reply other threads:[~2005-01-12 22:09 UTC|newest]
Thread overview: 21+ messages / expand[flat|nested] mbox.gz Atom feed top
2005-01-12 21:47 [PATCH][0/18] InfiniBand: updates for 2.6.11-rc1 Roland Dreier
2005-01-12 21:47 ` [PATCH][1/18] InfiniBand/IPoIB: use correct static rate in IpoIB Roland Dreier
2005-01-12 21:47 ` [PATCH][2/18] InfiniBand/mthca: trivial formatting fix Roland Dreier
2005-01-12 21:47 ` Roland Dreier [this message]
2005-01-12 21:47 ` [PATCH][4/18] InfiniBand/mthca: clean up allocation mapping of HCA context memory Roland Dreier
2005-01-12 21:47 ` [PATCH][5/18] InfiniBand/mthca: add needed rmb() in event queue poll Roland Dreier
2005-01-12 21:47 ` [PATCH][6/18] InfiniBand/core: remove debug printk Roland Dreier
2005-01-12 21:47 ` [PATCH][7/18] InfiniBand: make more code static Roland Dreier
2005-01-12 21:47 ` [PATCH][8/18] InfiniBand/core: set byte_cnt correctly in MAD completion Roland Dreier
2005-01-12 21:47 ` [PATCH][9/18] InfiniBand/core: add QP number to work completion struct Roland Dreier
2005-01-12 21:47 ` [PATCH][10/18] InfiniBand/core: add node_type and phys_state sysfs attrs Roland Dreier
2005-01-12 21:48 ` [PATCH][11/18] InfiniBand/mthca: clean up computation of HCA memory map Roland Dreier
2005-01-12 21:48 ` [PATCH][12/18] InfiniBand/core: fix handling of 0-hop directed route MADs Roland Dreier
2005-01-12 21:48 ` [PATCH][13/18] InfiniBand/core: add more parameters to process_mad Roland Dreier
2005-01-12 21:48 ` [PATCH][14/18] InfiniBand/core: add qp_type to struct ib_qp Roland Dreier
2005-01-12 21:48 ` [PATCH][15/18] InfiniBand/core: add ib_find_cached_gid function Roland Dreier
2005-01-12 21:48 ` [PATCH][16/18] InfiniBand: update copyrights for new year Roland Dreier
2005-01-12 21:48 ` [PATCH][17/18] InfiniBand/ipoib: move structs from stack to device private struct Roland Dreier
2005-01-12 21:48 ` [PATCH][18/18] InfiniBand/core: rename handle_outgoing_smp Roland Dreier
2005-01-13 9:45 ` [openib-general] [PATCH][5/18] InfiniBand/mthca: add needed rmb() in event queue poll Michael S. Tsirkin
2005-01-13 15:32 ` Roland Dreier
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20051121347.gzXWVDbfUCDwWmdf@topspin.com \
--to=roland@topspin.com \
--cc=akpm@osdl.org \
--cc=linux-kernel@vger.kernel.org \
--cc=openib-general@openib.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
Powered by JetHome