* [PATCH 01/10] staging: lustre: lnet: use struct lnet_ioctl_config_data for NI bring up
2016-05-07 1:30 [PATCH 00/10] staging: lustre: lnet: per NI map-on-demand value James Simmons
@ 2016-05-07 1:30 ` James Simmons
2016-05-07 1:30 ` [PATCH 02/10] staging: lustre: lnet: use struct lnet_ioctl_config_data for NI query James Simmons
` (8 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: James Simmons @ 2016-05-07 1:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, devel, Andreas Dilger, Oleg Drokin
Cc: Linux Kernel Mailing List, Lustre Development List, Amir Shehata,
James Simmons, James Simmons
From: Amir Shehata <amir.shehata@intel.com>
The LNet layer passes around the individual fields of struct
lnet_ioctl_config_data for the case of NI bring up. To simplify
the code lets just pass struct lnet_ioctl_config directly.
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Signed-off-by: James Simmons <uja.ornl@yahoo.com>
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-7101
Reviewed-on: http://review.whamcloud.com/16367
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Reviewed-by: Olaf Weber <olaf@sgi.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
.../staging/lustre/include/linux/lnet/lib-lnet.h | 5 +--
drivers/staging/lustre/lnet/lnet/api-ni.c | 37 +++++++++++--------
drivers/staging/lustre/lnet/lnet/module.c | 7 +---
3 files changed, 24 insertions(+), 25 deletions(-)
diff --git a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
index dfc0208..1865c85 100644
--- a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
@@ -478,9 +478,8 @@ int lnet_rtrpools_enable(void);
void lnet_rtrpools_disable(void);
void lnet_rtrpools_free(int keep_pools);
lnet_remotenet_t *lnet_find_net_locked(__u32 net);
-int lnet_dyn_add_ni(lnet_pid_t requested_pid, char *nets,
- __s32 peer_timeout, __s32 peer_cr, __s32 peer_buf_cr,
- __s32 credits);
+int lnet_dyn_add_ni(lnet_pid_t requested_pid,
+ struct lnet_ioctl_config_data *conf);
int lnet_dyn_del_ni(__u32 net);
int lnet_clear_lazy_portal(struct lnet_ni *ni, int portal, char *reason);
diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
index f825784..6340ad1 100644
--- a/drivers/staging/lustre/lnet/lnet/api-ni.c
+++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
@@ -1215,8 +1215,7 @@ lnet_shutdown_lndni(struct lnet_ni *ni)
}
static int
-lnet_startup_lndni(struct lnet_ni *ni, __s32 peer_timeout,
- __s32 peer_cr, __s32 peer_buf_cr, __s32 credits)
+lnet_startup_lndni(struct lnet_ni *ni, struct lnet_ioctl_config_data *conf)
{
int rc = -EINVAL;
int lnd_type;
@@ -1292,20 +1291,28 @@ lnet_startup_lndni(struct lnet_ni *ni, __s32 peer_timeout,
* If given some LND tunable parameters, parse those now to
* override the values in the NI structure.
*/
- if (peer_buf_cr >= 0)
- ni->ni_peerrtrcredits = peer_buf_cr;
- if (peer_timeout >= 0)
- ni->ni_peertimeout = peer_timeout;
+ if (conf && conf->cfg_config_u.cfg_net.net_peer_rtr_credits >= 0) {
+ ni->ni_peerrtrcredits =
+ conf->cfg_config_u.cfg_net.net_peer_rtr_credits;
+ }
+ if (conf && conf->cfg_config_u.cfg_net.net_peer_timeout >= 0) {
+ ni->ni_peertimeout =
+ conf->cfg_config_u.cfg_net.net_peer_timeout;
+ }
/*
* TODO
* Note: For now, don't allow the user to change
* peertxcredits as this number is used in the
* IB LND to control queue depth.
- * if (peer_cr != -1)
- * ni->ni_peertxcredits = peer_cr;
+ *
+ * if (conf && conf->cfg_config_u.cfg_net.net_peer_tx_credits != -1)
+ * ni->ni_peertxcredits =
+ * conf->cfg_config_u.cfg_net.net_peer_tx_credits;
*/
- if (credits >= 0)
- ni->ni_maxtxcredits = credits;
+ if (conf && conf->cfg_config_u.cfg_net.net_max_tx_credits >= 0) {
+ ni->ni_maxtxcredits =
+ conf->cfg_config_u.cfg_net.net_max_tx_credits;
+ }
LASSERT(ni->ni_peertimeout <= 0 || lnd->lnd_query);
@@ -1367,7 +1374,7 @@ lnet_startup_lndnis(struct list_head *nilist)
while (!list_empty(nilist)) {
ni = list_entry(nilist->next, lnet_ni_t, ni_list);
list_del(&ni->ni_list);
- rc = lnet_startup_lndni(ni, -1, -1, -1, -1);
+ rc = lnet_startup_lndni(ni, NULL);
if (rc < 0)
goto failed;
@@ -1725,10 +1732,9 @@ lnet_get_net_config(int idx, __u32 *cpt_count, __u64 *nid, int *peer_timeout,
}
int
-lnet_dyn_add_ni(lnet_pid_t requested_pid, char *nets,
- __s32 peer_timeout, __s32 peer_cr, __s32 peer_buf_cr,
- __s32 credits)
+lnet_dyn_add_ni(lnet_pid_t requested_pid, struct lnet_ioctl_config_data *conf)
{
+ char *nets = conf->cfg_config_u.cfg_net.net_intf;
lnet_ping_info_t *pinfo;
lnet_handle_md_t md_handle;
struct lnet_ni *ni;
@@ -1773,8 +1779,7 @@ lnet_dyn_add_ni(lnet_pid_t requested_pid, char *nets,
list_del_init(&ni->ni_list);
- rc = lnet_startup_lndni(ni, peer_timeout, peer_cr,
- peer_buf_cr, credits);
+ rc = lnet_startup_lndni(ni, conf);
if (rc)
goto failed1;
diff --git a/drivers/staging/lustre/lnet/lnet/module.c b/drivers/staging/lustre/lnet/lnet/module.c
index 93037c1..246b5c1 100644
--- a/drivers/staging/lustre/lnet/lnet/module.c
+++ b/drivers/staging/lustre/lnet/lnet/module.c
@@ -108,12 +108,7 @@ lnet_dyn_configure(struct libcfs_ioctl_hdr *hdr)
rc = -EINVAL;
goto out_unlock;
}
- rc = lnet_dyn_add_ni(LNET_PID_LUSTRE,
- conf->cfg_config_u.cfg_net.net_intf,
- conf->cfg_config_u.cfg_net.net_peer_timeout,
- conf->cfg_config_u.cfg_net.net_peer_tx_credits,
- conf->cfg_config_u.cfg_net.net_peer_rtr_credits,
- conf->cfg_config_u.cfg_net.net_max_tx_credits);
+ rc = lnet_dyn_add_ni(LNET_PID_LUSTRE, conf);
out_unlock:
mutex_unlock(&lnet_config_mutex);
--
1.7.1
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH 02/10] staging: lustre: lnet: use struct lnet_ioctl_config_data for NI query
2016-05-07 1:30 [PATCH 00/10] staging: lustre: lnet: per NI map-on-demand value James Simmons
2016-05-07 1:30 ` [PATCH 01/10] staging: lustre: lnet: use struct lnet_ioctl_config_data for NI bring up James Simmons
@ 2016-05-07 1:30 ` James Simmons
2016-05-07 1:30 ` [PATCH 03/10] staging: lustre: lnet: add per NI configuration structures James Simmons
` (7 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: James Simmons @ 2016-05-07 1:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, devel, Andreas Dilger, Oleg Drokin
Cc: Linux Kernel Mailing List, Lustre Development List, Amir Shehata,
James Simmons, James Simmons
From: Amir Shehata <amir.shehata@intel.com>
The functions used to query the NI state passes around the
individual fields of struct lnet_ioctl_config_data. To simplify
the code lets just pass struct lnet_ioctl_config directly.
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Signed-off-by: James Simmons <uja.ornl@yahoo.com>
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-7101
Reviewed-on: http://review.whamcloud.com/16367
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Reviewed-by: Olaf Weber <olaf@sgi.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
.../staging/lustre/include/linux/lnet/lib-lnet.h | 4 -
drivers/staging/lustre/lnet/lnet/api-ni.c | 60 ++++++--------------
2 files changed, 18 insertions(+), 46 deletions(-)
diff --git a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
index 1865c85..513a822 100644
--- a/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-lnet.h
@@ -463,10 +463,6 @@ int lnet_del_route(__u32 net, lnet_nid_t gw_nid);
void lnet_destroy_routes(void);
int lnet_get_route(int idx, __u32 *net, __u32 *hops,
lnet_nid_t *gateway, __u32 *alive, __u32 *priority);
-int lnet_get_net_config(int idx, __u32 *cpt_count, __u64 *nid,
- int *peer_timeout, int *peer_tx_credits,
- int *peer_rtr_cr, int *max_tx_credits,
- struct lnet_ioctl_net_config *net_config);
int lnet_get_rtr_pool_cfg(int idx, struct lnet_ioctl_pool_cfg *pool_cfg);
void lnet_router_debugfs_init(void);
diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
index 6340ad1..01e1808 100644
--- a/drivers/staging/lustre/lnet/lnet/api-ni.c
+++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
@@ -1648,25 +1648,18 @@ EXPORT_SYMBOL(LNetNIFini);
* parameters
*
* \param[in] ni network interface structure
- * \param[out] cpt_count the number of cpts the ni is on
- * \param[out] nid Network Interface ID
- * \param[out] peer_timeout NI peer timeout
- * \param[out] peer_tx_crdits NI peer transmit credits
- * \param[out] peer_rtr_credits NI peer router credits
- * \param[out] max_tx_credits NI max transmit credit
- * \param[out] net_config Network configuration
+ * \param[out] config NI configuration
*/
static void
-lnet_fill_ni_info(struct lnet_ni *ni, __u32 *cpt_count, __u64 *nid,
- int *peer_timeout, int *peer_tx_credits,
- int *peer_rtr_credits, int *max_tx_credits,
- struct lnet_ioctl_net_config *net_config)
+lnet_fill_ni_info(struct lnet_ni *ni, struct lnet_ioctl_config_data *config)
{
+ struct lnet_ioctl_net_config *net_config;
int i;
- if (!ni)
+ if (!ni || !config)
return;
+ net_config = (struct lnet_ioctl_net_config *) config->cfg_bulk;
if (!net_config)
return;
@@ -1682,11 +1675,11 @@ lnet_fill_ni_info(struct lnet_ni *ni, __u32 *cpt_count, __u64 *nid,
sizeof(net_config->ni_interfaces[i]));
}
- *nid = ni->ni_nid;
- *peer_timeout = ni->ni_peertimeout;
- *peer_tx_credits = ni->ni_peertxcredits;
- *peer_rtr_credits = ni->ni_peerrtrcredits;
- *max_tx_credits = ni->ni_maxtxcredits;
+ config->cfg_nid = ni->ni_nid;
+ config->cfg_config_u.cfg_net.net_peer_timeout = ni->ni_peertimeout;
+ config->cfg_config_u.cfg_net.net_max_tx_credits = ni->ni_maxtxcredits;
+ config->cfg_config_u.cfg_net.net_peer_tx_credits = ni->ni_peertxcredits;
+ config->cfg_config_u.cfg_net.net_peer_rtr_credits = ni->ni_peerrtrcredits;
net_config->ni_status = ni->ni_status->ns_status;
@@ -1696,18 +1689,16 @@ lnet_fill_ni_info(struct lnet_ni *ni, __u32 *cpt_count, __u64 *nid,
for (i = 0; i < num_cpts; i++)
net_config->ni_cpts[i] = ni->ni_cpts[i];
- *cpt_count = num_cpts;
+ config->cfg_ncpts = num_cpts;
}
}
-int
-lnet_get_net_config(int idx, __u32 *cpt_count, __u64 *nid, int *peer_timeout,
- int *peer_tx_credits, int *peer_rtr_credits,
- int *max_tx_credits,
- struct lnet_ioctl_net_config *net_config)
+static int
+lnet_get_net_config(struct lnet_ioctl_config_data *config)
{
struct lnet_ni *ni;
struct list_head *tmp;
+ int idx = config->cfg_count;
int cpt, i = 0;
int rc = -ENOENT;
@@ -1719,9 +1710,7 @@ lnet_get_net_config(int idx, __u32 *cpt_count, __u64 *nid, int *peer_timeout,
ni = list_entry(tmp, lnet_ni_t, ni_list);
lnet_ni_lock(ni);
- lnet_fill_ni_info(ni, cpt_count, nid, peer_timeout,
- peer_tx_credits, peer_rtr_credits,
- max_tx_credits, net_config);
+ lnet_fill_ni_info(ni, config);
lnet_ni_unlock(ni);
rc = 0;
break;
@@ -1927,27 +1916,14 @@ LNetCtl(unsigned int cmd, void *arg)
&config->cfg_config_u.cfg_route.rtr_priority);
case IOC_LIBCFS_GET_NET: {
- struct lnet_ioctl_net_config *net_config;
- size_t total = sizeof(*config) + sizeof(*net_config);
-
+ size_t total = sizeof(*config) +
+ sizeof(struct lnet_ioctl_net_config);
config = arg;
if (config->cfg_hdr.ioc_len < total)
return -EINVAL;
- net_config = (struct lnet_ioctl_net_config *)
- config->cfg_bulk;
- if (!net_config)
- return -EINVAL;
-
- return lnet_get_net_config(config->cfg_count,
- &config->cfg_ncpts,
- &config->cfg_nid,
- &config->cfg_config_u.cfg_net.net_peer_timeout,
- &config->cfg_config_u.cfg_net.net_peer_tx_credits,
- &config->cfg_config_u.cfg_net.net_peer_rtr_credits,
- &config->cfg_config_u.cfg_net.net_max_tx_credits,
- net_config);
+ return lnet_get_net_config(config);
}
case IOC_LIBCFS_GET_LNET_STATS: {
--
1.7.1
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH 03/10] staging: lustre: lnet: add per NI configuration structures
2016-05-07 1:30 [PATCH 00/10] staging: lustre: lnet: per NI map-on-demand value James Simmons
2016-05-07 1:30 ` [PATCH 01/10] staging: lustre: lnet: use struct lnet_ioctl_config_data for NI bring up James Simmons
2016-05-07 1:30 ` [PATCH 02/10] staging: lustre: lnet: use struct lnet_ioctl_config_data for NI query James Simmons
@ 2016-05-07 1:30 ` James Simmons
2016-05-07 1:30 ` [PATCH 04/10] staging: lustre: o2iblnd: convert macros to inline functions James Simmons
` (6 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: James Simmons @ 2016-05-07 1:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, devel, Andreas Dilger, Oleg Drokin
Cc: Linux Kernel Mailing List, Lustre Development List, Amir Shehata,
James Simmons, James Simmons
From: Amir Shehata <amir.shehata@intel.com>
Currently for LNet all hardware configuration is done with
the modprobe configuration file. These settings are applied
to all hardware instances for a node. That makes it impossible
to configure two different pieces of infiniband hardware that
require very different settings to function. Here we
introduce the infrastructre to allow applying per interface
settings.
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Signed-off-by: James Simmons <uja.ornl@yahoo.com>
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-7101
Reviewed-on: http://review.whamcloud.com/16367
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Reviewed-by: Olaf Weber <olaf@sgi.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
.../staging/lustre/include/linux/lnet/lib-dlc.h | 29 +++++++++++++++++++-
.../staging/lustre/include/linux/lnet/lib-types.h | 2 +
drivers/staging/lustre/lnet/lnet/config.c | 3 ++
3 files changed, 33 insertions(+), 1 deletions(-)
diff --git a/drivers/staging/lustre/include/linux/lnet/lib-dlc.h b/drivers/staging/lustre/include/linux/lnet/lib-dlc.h
index 84a19e9..6ce9acc 100644
--- a/drivers/staging/lustre/include/linux/lnet/lib-dlc.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-dlc.h
@@ -37,10 +37,37 @@
#define LNET_MAX_SHOW_NUM_CPT 128
#define LNET_UNDEFINED_HOPS ((__u32) -1)
+struct lnet_ioctl_config_lnd_cmn_tunables {
+ __u32 lct_version;
+ __u32 lct_peer_timeout;
+ __u32 lct_peer_tx_credits;
+ __u32 lct_peer_rtr_credits;
+ __u32 lct_max_tx_credits;
+};
+
+struct lnet_ioctl_config_o2iblnd_tunables {
+ __u32 lnd_version;
+ __u32 lnd_peercredits_hiw;
+ __u32 lnd_map_on_demand;
+ __u32 lnd_concurrent_sends;
+ __u32 lnd_fmr_pool_size;
+ __u32 lnd_fmr_flush_trigger;
+ __u32 lnd_fmr_cache;
+ __u32 pad;
+};
+
+struct lnet_ioctl_config_lnd_tunables {
+ struct lnet_ioctl_config_lnd_cmn_tunables lt_cmn;
+ union {
+ struct lnet_ioctl_config_o2iblnd_tunables lt_o2ib;
+ } lt_tun_u;
+};
+
struct lnet_ioctl_net_config {
char ni_interfaces[LNET_MAX_INTERFACES][LNET_MAX_STR_LEN];
__u32 ni_status;
__u32 ni_cpts[LNET_MAX_SHOW_NUM_CPT];
+ char cfg_bulk[0];
};
#define LNET_TINY_BUF_IDX 0
@@ -81,7 +108,7 @@ struct lnet_ioctl_config_data {
__s32 net_peer_rtr_credits;
__s32 net_max_tx_credits;
__u32 net_cksum_algo;
- __u32 net_pad;
+ __u32 net_interface_count;
} cfg_net;
struct {
__u32 buf_enable;
diff --git a/drivers/staging/lustre/include/linux/lnet/lib-types.h b/drivers/staging/lustre/include/linux/lnet/lib-types.h
index 29c72f8..24c4a08 100644
--- a/drivers/staging/lustre/include/linux/lnet/lib-types.h
+++ b/drivers/staging/lustre/include/linux/lnet/lib-types.h
@@ -273,6 +273,8 @@ typedef struct lnet_ni {
int **ni_refs; /* percpt reference count */
time64_t ni_last_alive;/* when I was last alive */
lnet_ni_status_t *ni_status; /* my health status */
+ /* per NI LND tunables */
+ struct lnet_ioctl_config_lnd_tunables *ni_lnd_tunables;
/* equivalent interfaces to use */
char *ni_interfaces[LNET_MAX_INTERFACES];
} lnet_ni_t;
diff --git a/drivers/staging/lustre/lnet/lnet/config.c b/drivers/staging/lustre/lnet/lnet/config.c
index 449069c..480cc9c 100644
--- a/drivers/staging/lustre/lnet/lnet/config.c
+++ b/drivers/staging/lustre/lnet/lnet/config.c
@@ -107,6 +107,9 @@ lnet_ni_free(struct lnet_ni *ni)
if (ni->ni_cpts)
cfs_expr_list_values_free(ni->ni_cpts, ni->ni_ncpts);
+ if (ni->ni_lnd_tunables)
+ LIBCFS_FREE(ni->ni_lnd_tunables, sizeof(*ni->ni_lnd_tunables));
+
for (i = 0; i < LNET_MAX_INTERFACES && ni->ni_interfaces[i]; i++) {
LIBCFS_FREE(ni->ni_interfaces[i],
strlen(ni->ni_interfaces[i]) + 1);
--
1.7.1
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH 04/10] staging: lustre: o2iblnd: convert macros to inline functions
2016-05-07 1:30 [PATCH 00/10] staging: lustre: lnet: per NI map-on-demand value James Simmons
` (2 preceding siblings ...)
2016-05-07 1:30 ` [PATCH 03/10] staging: lustre: lnet: add per NI configuration structures James Simmons
@ 2016-05-07 1:30 ` James Simmons
2016-05-07 1:30 ` [PATCH 05/10] staging: lustre: o2iblnd: create default o2iblnd tunable settings James Simmons
` (5 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: James Simmons @ 2016-05-07 1:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, devel, Andreas Dilger, Oleg Drokin
Cc: Linux Kernel Mailing List, Lustre Development List, Amir Shehata,
James Simmons, James Simmons
From: Amir Shehata <amir.shehata@intel.com>
Convert a few macros in o2iblnd.h to inline functions.
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Signed-off-by: James Simmons <uja.ornl@yahoo.com>
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-7101
Reviewed-on: http://review.whamcloud.com/16367
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Reviewed-by: Olaf Weber <olaf@sgi.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c | 2 +-
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h | 62 ++++++++++++--------
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c | 21 ++++---
.../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c | 11 ++++
4 files changed, 61 insertions(+), 35 deletions(-)
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
index ddc3313..a2e188e 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
@@ -335,7 +335,7 @@ int kiblnd_create_peer(lnet_ni_t *ni, kib_peer_t **peerp, lnet_nid_t nid)
peer->ibp_nid = nid;
peer->ibp_error = 0;
peer->ibp_last_alive = 0;
- peer->ibp_max_frags = IBLND_CFG_RDMA_FRAGS;
+ peer->ibp_max_frags = kiblnd_cfg_rdma_frags(peer->ibp_ni);
peer->ibp_queue_depth = *kiblnd_tunables.kib_peertxcredits;
atomic_set(&peer->ibp_refcount, 1); /* 1 ref for caller */
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
index 52245e0..fab0ef9 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
@@ -116,9 +116,6 @@ extern kib_tunables_t kiblnd_tunables;
#define IBLND_CREDITS_DEFAULT 8 /* default # of peer credits */
#define IBLND_CREDITS_MAX ((typeof(((kib_msg_t *) 0)->ibm_credits)) - 1) /* Max # of peer credits */
-#define IBLND_MSG_QUEUE_SIZE(v) ((v) == IBLND_MSG_VERSION_1 ? \
- IBLND_MSG_QUEUE_SIZE_V1 : \
- *kiblnd_tunables.kib_peertxcredits) /* # messages/RDMAs in-flight */
#define IBLND_CREDITS_HIGHWATER(v) ((v) == IBLND_MSG_VERSION_1 ? \
IBLND_CREDIT_HIGHWATER_V1 : \
*kiblnd_tunables.kib_peercredits_hiw) /* when eagerly to return credits */
@@ -127,32 +124,12 @@ extern kib_tunables_t kiblnd_tunables;
cb, dev, \
ps, qpt)
-static inline int
-kiblnd_concurrent_sends_v1(void)
-{
- if (*kiblnd_tunables.kib_concurrent_sends > IBLND_MSG_QUEUE_SIZE_V1 * 2)
- return IBLND_MSG_QUEUE_SIZE_V1 * 2;
-
- if (*kiblnd_tunables.kib_concurrent_sends < IBLND_MSG_QUEUE_SIZE_V1 / 2)
- return IBLND_MSG_QUEUE_SIZE_V1 / 2;
-
- return *kiblnd_tunables.kib_concurrent_sends;
-}
-
-#define IBLND_CONCURRENT_SENDS(v) ((v) == IBLND_MSG_VERSION_1 ? \
- kiblnd_concurrent_sends_v1() : \
- *kiblnd_tunables.kib_concurrent_sends)
/* 2 OOB shall suffice for 1 keepalive and 1 returning credits */
#define IBLND_OOB_CAPABLE(v) ((v) != IBLND_MSG_VERSION_1)
#define IBLND_OOB_MSGS(v) (IBLND_OOB_CAPABLE(v) ? 2 : 0)
#define IBLND_MSG_SIZE (4 << 10) /* max size of queued messages (inc hdr) */
#define IBLND_MAX_RDMA_FRAGS LNET_MAX_IOV /* max # of fragments supported */
-#define IBLND_CFG_RDMA_FRAGS (*kiblnd_tunables.kib_map_on_demand ? \
- *kiblnd_tunables.kib_map_on_demand : \
- IBLND_MAX_RDMA_FRAGS) /* max # of fragments configured by user */
-#define IBLND_RDMA_FRAGS(v) ((v) == IBLND_MSG_VERSION_1 ? \
- IBLND_MAX_RDMA_FRAGS : IBLND_CFG_RDMA_FRAGS)
/************************/
/* derived constants... */
@@ -171,7 +148,8 @@ kiblnd_concurrent_sends_v1(void)
/* WRs and CQEs (per connection) */
#define IBLND_RECV_WRS(c) IBLND_RX_MSGS(c)
#define IBLND_SEND_WRS(c) \
- ((c->ibc_max_frags + 1) * IBLND_CONCURRENT_SENDS(c->ibc_version))
+ ((c->ibc_max_frags + 1) * kiblnd_concurrent_sends(c->ibc_version, \
+ c->ibc_peer->ibp_ni))
#define IBLND_CQ_ENTRIES(c) (IBLND_RECV_WRS(c) + IBLND_SEND_WRS(c))
struct kib_hca_dev;
@@ -634,6 +612,42 @@ extern kib_data_t kiblnd_data;
void kiblnd_hdev_destroy(kib_hca_dev_t *hdev);
+int kiblnd_msg_queue_size(int version, struct lnet_ni *ni);
+
+/* max # of fragments configured by user */
+static inline int
+kiblnd_cfg_rdma_frags(struct lnet_ni *ni)
+{
+ int mod = *kiblnd_tunables.kib_map_on_demand;
+ return mod ? mod : IBLND_MAX_RDMA_FRAGS;
+}
+
+static inline int
+kiblnd_rdma_frags(int version, struct lnet_ni *ni)
+{
+ return version == IBLND_MSG_VERSION_1 ?
+ IBLND_MAX_RDMA_FRAGS :
+ kiblnd_cfg_rdma_frags(ni);
+}
+
+static inline int
+kiblnd_concurrent_sends(int version, struct lnet_ni *ni)
+{
+ int concurrent_sends;
+
+ concurrent_sends = *kiblnd_tunables.kib_concurrent_sends;
+
+ if (version == IBLND_MSG_VERSION_1) {
+ if (concurrent_sends > IBLND_MSG_QUEUE_SIZE_V1 * 2)
+ return IBLND_MSG_QUEUE_SIZE_V1 * 2;
+
+ if (concurrent_sends < IBLND_MSG_QUEUE_SIZE_V1 / 2)
+ return IBLND_MSG_QUEUE_SIZE_V1 / 2;
+ }
+
+ return concurrent_sends;
+}
+
static inline void
kiblnd_hdev_addref_locked(kib_hca_dev_t *hdev)
{
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
index 8bbd116..c3bd7fa 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -757,6 +757,7 @@ kiblnd_post_tx_locked(kib_conn_t *conn, kib_tx_t *tx, int credit)
{
kib_msg_t *msg = tx->tx_msg;
kib_peer_t *peer = conn->ibc_peer;
+ struct lnet_ni *ni = peer->ibp_ni;
int ver = conn->ibc_version;
int rc;
int done;
@@ -772,7 +773,7 @@ kiblnd_post_tx_locked(kib_conn_t *conn, kib_tx_t *tx, int credit)
LASSERT(conn->ibc_credits >= 0);
LASSERT(conn->ibc_credits <= conn->ibc_queue_depth);
- if (conn->ibc_nsends_posted == IBLND_CONCURRENT_SENDS(ver)) {
+ if (conn->ibc_nsends_posted == kiblnd_concurrent_sends(ver, ni)) {
/* tx completions outstanding... */
CDEBUG(D_NET, "%s: posted enough\n",
libcfs_nid2str(peer->ibp_nid));
@@ -923,7 +924,7 @@ kiblnd_check_sends(kib_conn_t *conn)
spin_lock(&conn->ibc_lock);
- LASSERT(conn->ibc_nsends_posted <= IBLND_CONCURRENT_SENDS(ver));
+ LASSERT(conn->ibc_nsends_posted <= kiblnd_concurrent_sends(ver, ni));
LASSERT(!IBLND_OOB_CAPABLE(ver) ||
conn->ibc_noops_posted <= IBLND_OOB_MSGS(ver));
LASSERT(conn->ibc_reserved_credits >= 0);
@@ -2337,11 +2338,11 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob)
}
if (reqmsg->ibm_u.connparams.ibcp_queue_depth >
- IBLND_MSG_QUEUE_SIZE(version)) {
+ kiblnd_msg_queue_size(version, ni)) {
CERROR("Can't accept conn from %s, queue depth too large: %d (<=%d wanted)\n",
libcfs_nid2str(nid),
reqmsg->ibm_u.connparams.ibcp_queue_depth,
- IBLND_MSG_QUEUE_SIZE(version));
+ kiblnd_msg_queue_size(version, ni));
if (version == IBLND_MSG_VERSION)
rej.ibr_why = IBLND_REJECT_MSG_QUEUE_SIZE;
@@ -2350,22 +2351,22 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob)
}
if (reqmsg->ibm_u.connparams.ibcp_max_frags >
- IBLND_RDMA_FRAGS(version)) {
+ kiblnd_rdma_frags(version, ni)) {
CWARN("Can't accept conn from %s (version %x): max_frags %d too large (%d wanted)\n",
libcfs_nid2str(nid), version,
reqmsg->ibm_u.connparams.ibcp_max_frags,
- IBLND_RDMA_FRAGS(version));
+ kiblnd_rdma_frags(version, ni));
if (version >= IBLND_MSG_VERSION)
rej.ibr_why = IBLND_REJECT_RDMA_FRAGS;
goto failed;
} else if (reqmsg->ibm_u.connparams.ibcp_max_frags <
- IBLND_RDMA_FRAGS(version) && !net->ibn_fmr_ps) {
+ kiblnd_rdma_frags(version, ni) && !net->ibn_fmr_ps) {
CWARN("Can't accept conn from %s (version %x): max_frags %d incompatible without FMR pool (%d wanted)\n",
libcfs_nid2str(nid), version,
reqmsg->ibm_u.connparams.ibcp_max_frags,
- IBLND_RDMA_FRAGS(version));
+ kiblnd_rdma_frags(version, ni));
if (version >= IBLND_MSG_VERSION)
rej.ibr_why = IBLND_REJECT_RDMA_FRAGS;
@@ -2532,8 +2533,8 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob)
lnet_ni_decref(ni);
rej.ibr_version = version;
- rej.ibr_cp.ibcp_queue_depth = IBLND_MSG_QUEUE_SIZE(version);
- rej.ibr_cp.ibcp_max_frags = IBLND_RDMA_FRAGS(version);
+ rej.ibr_cp.ibcp_queue_depth = kiblnd_msg_queue_size(version, ni);
+ rej.ibr_cp.ibcp_max_frags = kiblnd_rdma_frags(version, ni);
kiblnd_reject(cmid, &rej);
return -ECONNREFUSED;
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index b4607da..a6415fc 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -171,6 +171,17 @@ kib_tunables_t kiblnd_tunables = {
.kib_nscheds = &nscheds
};
+/* # messages/RDMAs in-flight */
+int kiblnd_msg_queue_size(int version, lnet_ni_t *ni)
+{
+ if (version == IBLND_MSG_VERSION_1)
+ return IBLND_MSG_QUEUE_SIZE_V1;
+ else if (ni)
+ return ni->ni_peertxcredits;
+ else
+ return peer_credits;
+}
+
int
kiblnd_tunables_init(void)
{
--
1.7.1
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH 05/10] staging: lustre: o2iblnd: create default o2iblnd tunable settings
2016-05-07 1:30 [PATCH 00/10] staging: lustre: lnet: per NI map-on-demand value James Simmons
` (3 preceding siblings ...)
2016-05-07 1:30 ` [PATCH 04/10] staging: lustre: o2iblnd: convert macros to inline functions James Simmons
@ 2016-05-07 1:30 ` James Simmons
2016-05-07 1:30 ` [PATCH 06/10] staging: lustre: o2iblnd: use data in lnet_ni_t instead of kiblnd_tunables James Simmons
` (4 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: James Simmons @ 2016-05-07 1:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, devel, Andreas Dilger, Oleg Drokin
Cc: Linux Kernel Mailing List, Lustre Development List, Amir Shehata,
James Simmons, James Simmons
From: Amir Shehata <amir.shehata@intel.com>
Create and set the default o2iblnd tunables when the ko2iblnd
module loads. Move kiblnd_tunables_setup() to when the NI
iterface is initialized.
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Signed-off-by: James Simmons <uja.ornl@yahoo.com>
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-7101
Reviewed-on: http://review.whamcloud.com/16367
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Reviewed-by: Olaf Weber <olaf@sgi.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c | 9 ++++-----
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h | 3 ++-
.../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c | 16 ++++++++++++++--
3 files changed, 20 insertions(+), 8 deletions(-)
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
index a2e188e..a06689b 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
@@ -2921,6 +2921,9 @@ static int kiblnd_startup(lnet_ni_t *ni)
net->ibn_incarnation = tv.tv_sec * USEC_PER_SEC +
tv.tv_nsec / NSEC_PER_USEC;
+ rc = kiblnd_tunables_setup();
+ if (rc)
+ goto net_failed;
ni->ni_peertimeout = *kiblnd_tunables.kib_peertimeout;
ni->ni_maxtxcredits = *kiblnd_tunables.kib_credits;
ni->ni_peertxcredits = *kiblnd_tunables.kib_peertxcredits;
@@ -3006,8 +3009,6 @@ static void __exit ko2iblnd_exit(void)
static int __init ko2iblnd_init(void)
{
- int rc;
-
CLASSERT(sizeof(kib_msg_t) <= IBLND_MSG_SIZE);
CLASSERT(offsetof(kib_msg_t,
ibm_u.get.ibgm_rd.rd_frags[IBLND_MAX_RDMA_FRAGS])
@@ -3016,9 +3017,7 @@ static int __init ko2iblnd_init(void)
ibm_u.putack.ibpam_rd.rd_frags[IBLND_MAX_RDMA_FRAGS])
<= IBLND_MSG_SIZE);
- rc = kiblnd_tunables_init();
- if (rc)
- return rc;
+ kiblnd_tunables_init();
lnet_register_lnd(&the_o2iblnd);
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
index fab0ef9..80e11bc 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
@@ -994,7 +994,8 @@ int kiblnd_fmr_pool_map(kib_fmr_poolset_t *fps, kib_tx_t *tx,
kib_fmr_t *fmr);
void kiblnd_fmr_pool_unmap(kib_fmr_t *fmr, int status);
-int kiblnd_tunables_init(void);
+int kiblnd_tunables_setup(void);
+void kiblnd_tunables_init(void);
void kiblnd_tunables_fini(void);
int kiblnd_connd(void *arg);
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index a6415fc..cc1469d 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -171,6 +171,8 @@ kib_tunables_t kiblnd_tunables = {
.kib_nscheds = &nscheds
};
+static struct lnet_ioctl_config_o2iblnd_tunables default_tunables;
+
/* # messages/RDMAs in-flight */
int kiblnd_msg_queue_size(int version, lnet_ni_t *ni)
{
@@ -182,8 +184,7 @@ int kiblnd_msg_queue_size(int version, lnet_ni_t *ni)
return peer_credits;
}
-int
-kiblnd_tunables_init(void)
+int kiblnd_tunables_setup(void)
{
if (kiblnd_translate_mtu(*kiblnd_tunables.kib_ib_mtu) < 0) {
CERROR("Invalid ib_mtu %d, expected 256/512/1024/2048/4096\n",
@@ -234,3 +235,14 @@ kiblnd_tunables_init(void)
return 0;
}
+
+void kiblnd_tunables_init(void)
+{
+ default_tunables.lnd_version = 0;
+ default_tunables.lnd_peercredits_hiw = peer_credits_hiw,
+ default_tunables.lnd_map_on_demand = map_on_demand;
+ default_tunables.lnd_concurrent_sends = concurrent_sends;
+ default_tunables.lnd_fmr_pool_size = fmr_pool_size;
+ default_tunables.lnd_fmr_flush_trigger = fmr_flush_trigger;
+ default_tunables.lnd_fmr_cache = fmr_cache;
+}
--
1.7.1
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH 06/10] staging: lustre: o2iblnd: use data in lnet_ni_t instead of kiblnd_tunables
2016-05-07 1:30 [PATCH 00/10] staging: lustre: lnet: per NI map-on-demand value James Simmons
` (4 preceding siblings ...)
2016-05-07 1:30 ` [PATCH 05/10] staging: lustre: o2iblnd: create default o2iblnd tunable settings James Simmons
@ 2016-05-07 1:30 ` James Simmons
2016-05-07 1:30 ` [PATCH 07/10] staging: lustre: o2iblnd: per NI map-on-demand value James Simmons
` (3 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: James Simmons @ 2016-05-07 1:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, devel, Andreas Dilger, Oleg Drokin
Cc: Linux Kernel Mailing List, Lustre Development List, Amir Shehata,
James Simmons, James Simmons
From: Amir Shehata <amir.shehata@intel.com>
Currently the ko2iblnd creates a kib_tunable_t stucture
to allow the ko2iblnd driver to access the module parameters
throught the code. Some of those data fields also exist
in lnet_ni_t. Migrate to using the lnet_ni_t data fields
instead of kib_tunable_t.
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Signed-off-by: James Simmons <uja.ornl@yahoo.com>
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-7101
Reviewed-on: http://review.whamcloud.com/16367
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Reviewed-by: Olaf Weber <olaf@sgi.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c | 8 +--
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h | 6 +--
.../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c | 54 +++++++++++--------
3 files changed, 34 insertions(+), 34 deletions(-)
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
index a06689b..84a15d4 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
@@ -336,7 +336,7 @@ int kiblnd_create_peer(lnet_ni_t *ni, kib_peer_t **peerp, lnet_nid_t nid)
peer->ibp_error = 0;
peer->ibp_last_alive = 0;
peer->ibp_max_frags = kiblnd_cfg_rdma_frags(peer->ibp_ni);
- peer->ibp_queue_depth = *kiblnd_tunables.kib_peertxcredits;
+ peer->ibp_queue_depth = ni->ni_peertxcredits;
atomic_set(&peer->ibp_refcount, 1); /* 1 ref for caller */
INIT_LIST_HEAD(&peer->ibp_list); /* not in the peer table yet */
@@ -2921,13 +2921,9 @@ static int kiblnd_startup(lnet_ni_t *ni)
net->ibn_incarnation = tv.tv_sec * USEC_PER_SEC +
tv.tv_nsec / NSEC_PER_USEC;
- rc = kiblnd_tunables_setup();
+ rc = kiblnd_tunables_setup(ni);
if (rc)
goto net_failed;
- ni->ni_peertimeout = *kiblnd_tunables.kib_peertimeout;
- ni->ni_maxtxcredits = *kiblnd_tunables.kib_credits;
- ni->ni_peertxcredits = *kiblnd_tunables.kib_peertxcredits;
- ni->ni_peerrtrcredits = *kiblnd_tunables.kib_peerrtrcredits;
if (ni->ni_interfaces[0]) {
/* Use the IPoIB interface specified in 'networks=' */
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
index 80e11bc..fffae0c 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
@@ -87,11 +87,7 @@ typedef struct {
int *kib_timeout; /* comms timeout (seconds) */
int *kib_keepalive; /* keepalive timeout (seconds) */
int *kib_ntx; /* # tx descs */
- int *kib_credits; /* # concurrent sends */
- int *kib_peertxcredits; /* # concurrent sends to 1 peer */
- int *kib_peerrtrcredits; /* # per-peer router buffer credits */
int *kib_peercredits_hiw; /* # when eagerly to return credits */
- int *kib_peertimeout; /* seconds to consider peer dead */
char **kib_default_ipif; /* default IPoIB interface */
int *kib_retry_count;
int *kib_rnr_retry_count;
@@ -994,7 +990,7 @@ int kiblnd_fmr_pool_map(kib_fmr_poolset_t *fps, kib_tx_t *tx,
kib_fmr_t *fmr);
void kiblnd_fmr_pool_unmap(kib_fmr_t *fmr, int status);
-int kiblnd_tunables_setup(void);
+int kiblnd_tunables_setup(struct lnet_ni *ni);
void kiblnd_tunables_init(void);
void kiblnd_tunables_fini(void);
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index cc1469d..e50a9cf 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -152,11 +152,7 @@ kib_tunables_t kiblnd_tunables = {
.kib_timeout = &timeout,
.kib_keepalive = &keepalive,
.kib_ntx = &ntx,
- .kib_credits = &credits,
- .kib_peertxcredits = &peer_credits,
.kib_peercredits_hiw = &peer_credits_hiw,
- .kib_peerrtrcredits = &peer_buffer_credits,
- .kib_peertimeout = &peer_timeout,
.kib_default_ipif = &ipif_name,
.kib_retry_count = &retry_count,
.kib_rnr_retry_count = &rnr_retry_count,
@@ -184,7 +180,7 @@ int kiblnd_msg_queue_size(int version, lnet_ni_t *ni)
return peer_credits;
}
-int kiblnd_tunables_setup(void)
+int kiblnd_tunables_setup(struct lnet_ni *ni)
{
if (kiblnd_translate_mtu(*kiblnd_tunables.kib_ib_mtu) < 0) {
CERROR("Invalid ib_mtu %d, expected 256/512/1024/2048/4096\n",
@@ -192,20 +188,32 @@ int kiblnd_tunables_setup(void)
return -EINVAL;
}
- if (*kiblnd_tunables.kib_peertxcredits < IBLND_CREDITS_DEFAULT)
- *kiblnd_tunables.kib_peertxcredits = IBLND_CREDITS_DEFAULT;
+ if (!ni->ni_peertimeout)
+ ni->ni_peertimeout = peer_timeout;
- if (*kiblnd_tunables.kib_peertxcredits > IBLND_CREDITS_MAX)
- *kiblnd_tunables.kib_peertxcredits = IBLND_CREDITS_MAX;
+ if (!ni->ni_maxtxcredits)
+ ni->ni_maxtxcredits = credits;
- if (*kiblnd_tunables.kib_peertxcredits > *kiblnd_tunables.kib_credits)
- *kiblnd_tunables.kib_peertxcredits = *kiblnd_tunables.kib_credits;
+ if (!ni->ni_peertxcredits)
+ ni->ni_peertxcredits = peer_credits;
- if (*kiblnd_tunables.kib_peercredits_hiw < *kiblnd_tunables.kib_peertxcredits / 2)
- *kiblnd_tunables.kib_peercredits_hiw = *kiblnd_tunables.kib_peertxcredits / 2;
+ if (!ni->ni_peerrtrcredits)
+ ni->ni_peerrtrcredits = peer_buffer_credits;
- if (*kiblnd_tunables.kib_peercredits_hiw >= *kiblnd_tunables.kib_peertxcredits)
- *kiblnd_tunables.kib_peercredits_hiw = *kiblnd_tunables.kib_peertxcredits - 1;
+ if (ni->ni_peertxcredits < IBLND_CREDITS_DEFAULT)
+ ni->ni_peertxcredits = IBLND_CREDITS_DEFAULT;
+
+ if (ni->ni_peertxcredits > IBLND_CREDITS_MAX)
+ ni->ni_peertxcredits = IBLND_CREDITS_MAX;
+
+ if (ni->ni_peertxcredits > credits)
+ ni->ni_peertxcredits = credits;
+
+ if (*kiblnd_tunables.kib_peercredits_hiw < ni->ni_peertxcredits / 2)
+ *kiblnd_tunables.kib_peercredits_hiw = ni->ni_peertxcredits / 2;
+
+ if (*kiblnd_tunables.kib_peercredits_hiw >= ni->ni_peertxcredits)
+ *kiblnd_tunables.kib_peercredits_hiw = ni->ni_peertxcredits - 1;
if (*kiblnd_tunables.kib_map_on_demand < 0 ||
*kiblnd_tunables.kib_map_on_demand > IBLND_MAX_RDMA_FRAGS)
@@ -217,20 +225,20 @@ int kiblnd_tunables_setup(void)
if (!*kiblnd_tunables.kib_concurrent_sends) {
if (*kiblnd_tunables.kib_map_on_demand > 0 &&
*kiblnd_tunables.kib_map_on_demand <= IBLND_MAX_RDMA_FRAGS / 8)
- *kiblnd_tunables.kib_concurrent_sends = (*kiblnd_tunables.kib_peertxcredits) * 2;
+ *kiblnd_tunables.kib_concurrent_sends = ni->ni_peertxcredits * 2;
else
- *kiblnd_tunables.kib_concurrent_sends = (*kiblnd_tunables.kib_peertxcredits);
+ *kiblnd_tunables.kib_concurrent_sends = ni->ni_peertxcredits;
}
- if (*kiblnd_tunables.kib_concurrent_sends > *kiblnd_tunables.kib_peertxcredits * 2)
- *kiblnd_tunables.kib_concurrent_sends = *kiblnd_tunables.kib_peertxcredits * 2;
+ if (*kiblnd_tunables.kib_concurrent_sends > ni->ni_peertxcredits * 2)
+ *kiblnd_tunables.kib_concurrent_sends = ni->ni_peertxcredits * 2;
- if (*kiblnd_tunables.kib_concurrent_sends < *kiblnd_tunables.kib_peertxcredits / 2)
- *kiblnd_tunables.kib_concurrent_sends = *kiblnd_tunables.kib_peertxcredits / 2;
+ if (*kiblnd_tunables.kib_concurrent_sends < ni->ni_peertxcredits / 2)
+ *kiblnd_tunables.kib_concurrent_sends = ni->ni_peertxcredits / 2;
- if (*kiblnd_tunables.kib_concurrent_sends < *kiblnd_tunables.kib_peertxcredits) {
+ if (*kiblnd_tunables.kib_concurrent_sends < ni->ni_peertxcredits) {
CWARN("Concurrent sends %d is lower than message queue size: %d, performance may drop slightly.\n",
- *kiblnd_tunables.kib_concurrent_sends, *kiblnd_tunables.kib_peertxcredits);
+ *kiblnd_tunables.kib_concurrent_sends, ni->ni_peertxcredits);
}
return 0;
--
1.7.1
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH 07/10] staging: lustre: o2iblnd: per NI map-on-demand value
2016-05-07 1:30 [PATCH 00/10] staging: lustre: lnet: per NI map-on-demand value James Simmons
` (5 preceding siblings ...)
2016-05-07 1:30 ` [PATCH 06/10] staging: lustre: o2iblnd: use data in lnet_ni_t instead of kiblnd_tunables James Simmons
@ 2016-05-07 1:30 ` James Simmons
2016-05-07 1:30 ` [PATCH 08/10] staging: lustre: o2iblnd: remove last of kiblnd_tunables_fini James Simmons
` (2 subsequent siblings)
9 siblings, 0 replies; 11+ messages in thread
From: James Simmons @ 2016-05-07 1:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, devel, Andreas Dilger, Oleg Drokin
Cc: Linux Kernel Mailing List, Lustre Development List, Amir Shehata,
James Simmons, James Simmons
From: Amir Shehata <amir.shehata@intel.com>
Enables support of different map-on-demand values per NI. This is
required to support OPA coexistence with MLX5 cards. MLX5 does not
support FMR, which is enabled via map-on-demand. However OPA's
performance is greatly enahanced when FMR is enabled. In order
to enable coexistence of both of these two types of cards we
need to be able to set different map-on-demand values for both NIs.
This patch also lays the ground work for other per NI tunables to
be added in future patches.
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Signed-off-by: James Simmons <uja.ornl@yahoo.com>
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-7101
Reviewed-on: http://review.whamcloud.com/16367
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Reviewed-by: Olaf Weber <olaf@sgi.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c | 66 ++++++++++------
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h | 35 +++++----
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c | 13 ++-
.../lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c | 84 +++++++++++++------
4 files changed, 125 insertions(+), 73 deletions(-)
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
index 84a15d4..7711d96 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.c
@@ -1283,16 +1283,22 @@ static void kiblnd_map_tx_pool(kib_tx_pool_t *tpo)
}
}
-struct ib_mr *kiblnd_find_rd_dma_mr(kib_hca_dev_t *hdev, kib_rdma_desc_t *rd,
+struct ib_mr *kiblnd_find_rd_dma_mr(struct lnet_ni *ni, kib_rdma_desc_t *rd,
int negotiated_nfrags)
{
- __u16 nfrags = (negotiated_nfrags != -1) ?
- negotiated_nfrags : *kiblnd_tunables.kib_map_on_demand;
+ kib_net_t *net = ni->ni_data;
+ kib_hca_dev_t *hdev = net->ibn_dev->ibd_hdev;
+ struct lnet_ioctl_config_o2iblnd_tunables *tunables;
+ __u16 nfrags;
+ int mod;
+
+ tunables = &ni->ni_lnd_tunables->lt_tun_u.lt_o2ib;
+ mod = tunables->lnd_map_on_demand;
+ nfrags = (negotiated_nfrags != -1) ? negotiated_nfrags : mod;
LASSERT(hdev->ibh_mrs);
- if (*kiblnd_tunables.kib_map_on_demand > 0 &&
- nfrags <= rd->rd_nfrags)
+ if (mod > 0 && nfrags <= rd->rd_nfrags)
return NULL;
return hdev->ibh_mrs;
@@ -1337,16 +1343,20 @@ static void kiblnd_destroy_fmr_pool_list(struct list_head *head)
}
}
-static int kiblnd_fmr_pool_size(int ncpts)
+static int
+kiblnd_fmr_pool_size(struct lnet_ioctl_config_o2iblnd_tunables *tunables,
+ int ncpts)
{
- int size = *kiblnd_tunables.kib_fmr_pool_size / ncpts;
+ int size = tunables->lnd_fmr_pool_size / ncpts;
return max(IBLND_FMR_POOL, size);
}
-static int kiblnd_fmr_flush_trigger(int ncpts)
+static int
+kiblnd_fmr_flush_trigger(struct lnet_ioctl_config_o2iblnd_tunables *tunables,
+ int ncpts)
{
- int size = *kiblnd_tunables.kib_fmr_flush_trigger / ncpts;
+ int size = tunables->lnd_fmr_flush_trigger / ncpts;
return max(IBLND_FMR_POOL_FLUSH, size);
}
@@ -1362,7 +1372,7 @@ static int kiblnd_alloc_fmr_pool(kib_fmr_poolset_t *fps, kib_fmr_pool_t *fpo)
.dirty_watermark = fps->fps_flush_trigger,
.flush_function = NULL,
.flush_arg = NULL,
- .cache = !!*kiblnd_tunables.kib_fmr_cache};
+ .cache = !!fps->fps_cache };
int rc = 0;
fpo->fmr.fpo_fmr_pool = ib_create_fmr_pool(fpo->fpo_hdev->ibh_pd,
@@ -1508,9 +1518,10 @@ static void kiblnd_fini_fmr_poolset(kib_fmr_poolset_t *fps)
}
}
-static int kiblnd_init_fmr_poolset(kib_fmr_poolset_t *fps, int cpt,
- kib_net_t *net, int pool_size,
- int flush_trigger)
+static int
+kiblnd_init_fmr_poolset(kib_fmr_poolset_t *fps, int cpt, int ncpts,
+ kib_net_t *net,
+ struct lnet_ioctl_config_o2iblnd_tunables *tunables)
{
kib_fmr_pool_t *fpo;
int rc;
@@ -1519,8 +1530,11 @@ static int kiblnd_init_fmr_poolset(kib_fmr_poolset_t *fps, int cpt,
fps->fps_net = net;
fps->fps_cpt = cpt;
- fps->fps_pool_size = pool_size;
- fps->fps_flush_trigger = flush_trigger;
+
+ fps->fps_pool_size = kiblnd_fmr_pool_size(tunables, ncpts);
+ fps->fps_flush_trigger = kiblnd_fmr_flush_trigger(tunables, ncpts);
+ fps->fps_cache = tunables->lnd_fmr_cache;
+
spin_lock_init(&fps->fps_lock);
INIT_LIST_HEAD(&fps->fps_pool_list);
INIT_LIST_HEAD(&fps->fps_failed_pool_list);
@@ -2151,25 +2165,28 @@ static void kiblnd_net_fini_pools(kib_net_t *net)
}
}
-static int kiblnd_net_init_pools(kib_net_t *net, __u32 *cpts, int ncpts)
+static int kiblnd_net_init_pools(kib_net_t *net, lnet_ni_t *ni, __u32 *cpts,
+ int ncpts)
{
+ struct lnet_ioctl_config_o2iblnd_tunables *tunables;
unsigned long flags;
int cpt;
- int rc = 0;
+ int rc;
int i;
+ tunables = &ni->ni_lnd_tunables->lt_tun_u.lt_o2ib;
+
read_lock_irqsave(&kiblnd_data.kib_global_lock, flags);
- if (!*kiblnd_tunables.kib_map_on_demand) {
+ if (!tunables->lnd_map_on_demand) {
read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
goto create_tx_pool;
}
read_unlock_irqrestore(&kiblnd_data.kib_global_lock, flags);
- if (*kiblnd_tunables.kib_fmr_pool_size <
- *kiblnd_tunables.kib_ntx / 4) {
+ if (tunables->lnd_fmr_pool_size < *kiblnd_tunables.kib_ntx / 4) {
CERROR("Can't set fmr pool size (%d) < ntx / 4(%d)\n",
- *kiblnd_tunables.kib_fmr_pool_size,
+ tunables->lnd_fmr_pool_size,
*kiblnd_tunables.kib_ntx / 4);
rc = -EINVAL;
goto failed;
@@ -2199,9 +2216,8 @@ static int kiblnd_net_init_pools(kib_net_t *net, __u32 *cpts, int ncpts)
for (i = 0; i < ncpts; i++) {
cpt = !cpts ? i : cpts[i];
- rc = kiblnd_init_fmr_poolset(net->ibn_fmr_ps[cpt], cpt, net,
- kiblnd_fmr_pool_size(ncpts),
- kiblnd_fmr_flush_trigger(ncpts));
+ rc = kiblnd_init_fmr_poolset(net->ibn_fmr_ps[cpt], cpt, ncpts,
+ net, tunables);
if (rc) {
CERROR("Can't initialize FMR pool for CPT %d: %d\n",
cpt, rc);
@@ -2962,7 +2978,7 @@ static int kiblnd_startup(lnet_ni_t *ni)
if (rc)
goto failed;
- rc = kiblnd_net_init_pools(net, ni->ni_cpts, ni->ni_ncpts);
+ rc = kiblnd_net_init_pools(net, ni, ni->ni_cpts, ni->ni_ncpts);
if (rc) {
CERROR("Failed to initialize NI pools: %d\n", rc);
goto failed;
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
index fffae0c..d458773 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
@@ -87,18 +87,10 @@ typedef struct {
int *kib_timeout; /* comms timeout (seconds) */
int *kib_keepalive; /* keepalive timeout (seconds) */
int *kib_ntx; /* # tx descs */
- int *kib_peercredits_hiw; /* # when eagerly to return credits */
char **kib_default_ipif; /* default IPoIB interface */
int *kib_retry_count;
int *kib_rnr_retry_count;
- int *kib_concurrent_sends; /* send work queue sizing */
int *kib_ib_mtu; /* IB MTU */
- int *kib_map_on_demand; /* map-on-demand if RD has more */
- /* fragments than this value, 0 */
- /* disable map-on-demand */
- int *kib_fmr_pool_size; /* # FMRs in pool */
- int *kib_fmr_flush_trigger; /* When to trigger FMR flush */
- int *kib_fmr_cache; /* enable FMR pool cache? */
int *kib_require_priv_port; /* accept only privileged ports */
int *kib_use_priv_port; /* use privileged port for active connect */
int *kib_nscheds; /* # threads on each CPT */
@@ -112,9 +104,10 @@ extern kib_tunables_t kiblnd_tunables;
#define IBLND_CREDITS_DEFAULT 8 /* default # of peer credits */
#define IBLND_CREDITS_MAX ((typeof(((kib_msg_t *) 0)->ibm_credits)) - 1) /* Max # of peer credits */
-#define IBLND_CREDITS_HIGHWATER(v) ((v) == IBLND_MSG_VERSION_1 ? \
- IBLND_CREDIT_HIGHWATER_V1 : \
- *kiblnd_tunables.kib_peercredits_hiw) /* when eagerly to return credits */
+/* when eagerly to return credits */
+#define IBLND_CREDITS_HIGHWATER(t, v) ((v) == IBLND_MSG_VERSION_1 ? \
+ IBLND_CREDIT_HIGHWATER_V1 : \
+ t->lnd_peercredits_hiw)
#define kiblnd_rdma_create_id(cb, dev, ps, qpt) rdma_create_id(&init_net, \
cb, dev, \
@@ -260,6 +253,7 @@ typedef struct {
int fps_cpt; /* CPT id */
int fps_pool_size;
int fps_flush_trigger;
+ int fps_cache;
int fps_increasing; /* is allocating new pool */
unsigned long fps_next_retry; /* time stamp for retry if*/
/* failed to allocate */
@@ -614,7 +608,11 @@ int kiblnd_msg_queue_size(int version, struct lnet_ni *ni);
static inline int
kiblnd_cfg_rdma_frags(struct lnet_ni *ni)
{
- int mod = *kiblnd_tunables.kib_map_on_demand;
+ struct lnet_ioctl_config_o2iblnd_tunables *tunables;
+ int mod;
+
+ tunables = &ni->ni_lnd_tunables->lt_tun_u.lt_o2ib;
+ mod = tunables->lnd_map_on_demand;
return mod ? mod : IBLND_MAX_RDMA_FRAGS;
}
@@ -629,9 +627,11 @@ kiblnd_rdma_frags(int version, struct lnet_ni *ni)
static inline int
kiblnd_concurrent_sends(int version, struct lnet_ni *ni)
{
+ struct lnet_ioctl_config_o2iblnd_tunables *tunables;
int concurrent_sends;
- concurrent_sends = *kiblnd_tunables.kib_concurrent_sends;
+ tunables = &ni->ni_lnd_tunables->lt_tun_u.lt_o2ib;
+ concurrent_sends = tunables->lnd_concurrent_sends;
if (version == IBLND_MSG_VERSION_1) {
if (concurrent_sends > IBLND_MSG_QUEUE_SIZE_V1 * 2)
@@ -766,10 +766,14 @@ kiblnd_send_keepalive(kib_conn_t *conn)
static inline int
kiblnd_need_noop(kib_conn_t *conn)
{
+ struct lnet_ioctl_config_o2iblnd_tunables *tunables;
+ lnet_ni_t *ni = conn->ibc_peer->ibp_ni;
+
LASSERT(conn->ibc_state >= IBLND_CONN_ESTABLISHED);
+ tunables = &ni->ni_lnd_tunables->lt_tun_u.lt_o2ib;
if (conn->ibc_outstanding_credits <
- IBLND_CREDITS_HIGHWATER(conn->ibc_version) &&
+ IBLND_CREDITS_HIGHWATER(tunables, conn->ibc_version) &&
!kiblnd_send_keepalive(conn))
return 0; /* No need to send NOOP */
@@ -977,8 +981,7 @@ static inline unsigned int kiblnd_sg_dma_len(struct ib_device *dev,
#define KIBLND_CONN_PARAM(e) ((e)->param.conn.private_data)
#define KIBLND_CONN_PARAM_LEN(e) ((e)->param.conn.private_data_len)
-struct ib_mr *kiblnd_find_rd_dma_mr(kib_hca_dev_t *hdev,
- kib_rdma_desc_t *rd,
+struct ib_mr *kiblnd_find_rd_dma_mr(struct lnet_ni *ni, kib_rdma_desc_t *rd,
int negotiated_nfrags);
void kiblnd_map_rx_descs(kib_conn_t *conn);
void kiblnd_unmap_rx_descs(kib_conn_t *conn);
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
index c3bd7fa..c4daf95 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -612,8 +612,8 @@ static void kiblnd_unmap_tx(lnet_ni_t *ni, kib_tx_t *tx)
static int kiblnd_map_tx(lnet_ni_t *ni, kib_tx_t *tx, kib_rdma_desc_t *rd,
int nfrags)
{
- kib_hca_dev_t *hdev = tx->tx_pool->tpo_hdev;
kib_net_t *net = ni->ni_data;
+ kib_hca_dev_t *hdev = net->ibn_dev->ibd_hdev;
struct ib_mr *mr = NULL;
__u32 nob;
int i;
@@ -636,7 +636,7 @@ static int kiblnd_map_tx(lnet_ni_t *ni, kib_tx_t *tx, kib_rdma_desc_t *rd,
nob += rd->rd_frags[i].rf_nob;
}
- mr = kiblnd_find_rd_dma_mr(hdev, rd, tx->tx_conn ?
+ mr = kiblnd_find_rd_dma_mr(ni, rd, tx->tx_conn ?
tx->tx_conn->ibc_max_frags : -1);
if (mr) {
/* found pre-mapping MR */
@@ -2585,12 +2585,15 @@ kiblnd_check_reconnect(kib_conn_t *conn, int version,
reason = "Unknown";
break;
- case IBLND_REJECT_RDMA_FRAGS:
+ case IBLND_REJECT_RDMA_FRAGS: {
+ struct lnet_ioctl_config_lnd_tunables *tunables;
+
if (!cp) {
reason = "can't negotiate max frags";
goto out;
}
- if (!*kiblnd_tunables.kib_map_on_demand) {
+ tunables = peer->ibp_ni->ni_lnd_tunables;
+ if (!tunables->lt_tun_u.lt_o2ib.lnd_map_on_demand) {
reason = "map_on_demand must be enabled";
goto out;
}
@@ -2602,7 +2605,7 @@ kiblnd_check_reconnect(kib_conn_t *conn, int version,
peer->ibp_max_frags = frag_num;
reason = "rdma fragments";
break;
-
+ }
case IBLND_REJECT_MSG_QUEUE_SIZE:
if (!cp) {
reason = "can't negotiate queue depth";
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c
index e50a9cf..f8fdd4a 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_modparams.c
@@ -152,16 +152,10 @@ kib_tunables_t kiblnd_tunables = {
.kib_timeout = &timeout,
.kib_keepalive = &keepalive,
.kib_ntx = &ntx,
- .kib_peercredits_hiw = &peer_credits_hiw,
.kib_default_ipif = &ipif_name,
.kib_retry_count = &retry_count,
.kib_rnr_retry_count = &rnr_retry_count,
- .kib_concurrent_sends = &concurrent_sends,
.kib_ib_mtu = &ib_mtu,
- .kib_map_on_demand = &map_on_demand,
- .kib_fmr_pool_size = &fmr_pool_size,
- .kib_fmr_flush_trigger = &fmr_flush_trigger,
- .kib_fmr_cache = &fmr_cache,
.kib_require_priv_port = &require_privileged_port,
.kib_use_priv_port = &use_privileged_port,
.kib_nscheds = &nscheds
@@ -182,6 +176,26 @@ int kiblnd_msg_queue_size(int version, lnet_ni_t *ni)
int kiblnd_tunables_setup(struct lnet_ni *ni)
{
+ struct lnet_ioctl_config_o2iblnd_tunables *tunables;
+
+ /*
+ * if there was no tunables specified, setup the tunables to be
+ * defaulted
+ */
+ if (!ni->ni_lnd_tunables) {
+ LIBCFS_ALLOC(ni->ni_lnd_tunables,
+ sizeof(*ni->ni_lnd_tunables));
+ if (!ni->ni_lnd_tunables)
+ return -ENOMEM;
+
+ memcpy(&ni->ni_lnd_tunables->lt_tun_u.lt_o2ib,
+ &default_tunables, sizeof(*tunables));
+ }
+ tunables = &ni->ni_lnd_tunables->lt_tun_u.lt_o2ib;
+
+ /* Current API version */
+ tunables->lnd_version = 0;
+
if (kiblnd_translate_mtu(*kiblnd_tunables.kib_ib_mtu) < 0) {
CERROR("Invalid ib_mtu %d, expected 256/512/1024/2048/4096\n",
*kiblnd_tunables.kib_ib_mtu);
@@ -209,38 +223,54 @@ int kiblnd_tunables_setup(struct lnet_ni *ni)
if (ni->ni_peertxcredits > credits)
ni->ni_peertxcredits = credits;
- if (*kiblnd_tunables.kib_peercredits_hiw < ni->ni_peertxcredits / 2)
- *kiblnd_tunables.kib_peercredits_hiw = ni->ni_peertxcredits / 2;
+ if (!tunables->lnd_peercredits_hiw)
+ tunables->lnd_peercredits_hiw = peer_credits_hiw;
- if (*kiblnd_tunables.kib_peercredits_hiw >= ni->ni_peertxcredits)
- *kiblnd_tunables.kib_peercredits_hiw = ni->ni_peertxcredits - 1;
+ if (tunables->lnd_peercredits_hiw < ni->ni_peertxcredits / 2)
+ tunables->lnd_peercredits_hiw = ni->ni_peertxcredits / 2;
- if (*kiblnd_tunables.kib_map_on_demand < 0 ||
- *kiblnd_tunables.kib_map_on_demand > IBLND_MAX_RDMA_FRAGS)
- *kiblnd_tunables.kib_map_on_demand = 0; /* disable map-on-demand */
+ if (tunables->lnd_peercredits_hiw >= ni->ni_peertxcredits)
+ tunables->lnd_peercredits_hiw = ni->ni_peertxcredits - 1;
- if (*kiblnd_tunables.kib_map_on_demand == 1)
- *kiblnd_tunables.kib_map_on_demand = 2; /* don't make sense to create map if only one fragment */
+ if (tunables->lnd_map_on_demand < 0 ||
+ tunables->lnd_map_on_demand > IBLND_MAX_RDMA_FRAGS) {
+ /* disable map-on-demand */
+ tunables->lnd_map_on_demand = 0;
+ }
- if (!*kiblnd_tunables.kib_concurrent_sends) {
- if (*kiblnd_tunables.kib_map_on_demand > 0 &&
- *kiblnd_tunables.kib_map_on_demand <= IBLND_MAX_RDMA_FRAGS / 8)
- *kiblnd_tunables.kib_concurrent_sends = ni->ni_peertxcredits * 2;
- else
- *kiblnd_tunables.kib_concurrent_sends = ni->ni_peertxcredits;
+ if (tunables->lnd_map_on_demand == 1) {
+ /* don't make sense to create map if only one fragment */
+ tunables->lnd_map_on_demand = 2;
}
- if (*kiblnd_tunables.kib_concurrent_sends > ni->ni_peertxcredits * 2)
- *kiblnd_tunables.kib_concurrent_sends = ni->ni_peertxcredits * 2;
+ if (!tunables->lnd_concurrent_sends) {
+ if (tunables->lnd_map_on_demand > 0 &&
+ tunables->lnd_map_on_demand <= IBLND_MAX_RDMA_FRAGS / 8) {
+ tunables->lnd_concurrent_sends =
+ ni->ni_peertxcredits * 2;
+ } else {
+ tunables->lnd_concurrent_sends = ni->ni_peertxcredits;
+ }
+ }
+
+ if (tunables->lnd_concurrent_sends > ni->ni_peertxcredits * 2)
+ tunables->lnd_concurrent_sends = ni->ni_peertxcredits * 2;
- if (*kiblnd_tunables.kib_concurrent_sends < ni->ni_peertxcredits / 2)
- *kiblnd_tunables.kib_concurrent_sends = ni->ni_peertxcredits / 2;
+ if (tunables->lnd_concurrent_sends < ni->ni_peertxcredits / 2)
+ tunables->lnd_concurrent_sends = ni->ni_peertxcredits / 2;
- if (*kiblnd_tunables.kib_concurrent_sends < ni->ni_peertxcredits) {
+ if (tunables->lnd_concurrent_sends < ni->ni_peertxcredits) {
CWARN("Concurrent sends %d is lower than message queue size: %d, performance may drop slightly.\n",
- *kiblnd_tunables.kib_concurrent_sends, ni->ni_peertxcredits);
+ tunables->lnd_concurrent_sends, ni->ni_peertxcredits);
}
+ if (!tunables->lnd_fmr_pool_size)
+ tunables->lnd_fmr_pool_size = fmr_pool_size;
+ if (!tunables->lnd_fmr_flush_trigger)
+ tunables->lnd_fmr_flush_trigger = fmr_flush_trigger;
+ if (!tunables->lnd_fmr_cache)
+ tunables->lnd_fmr_cache = fmr_cache;
+
return 0;
}
--
1.7.1
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH 08/10] staging: lustre: o2iblnd: remove last of kiblnd_tunables_fini
2016-05-07 1:30 [PATCH 00/10] staging: lustre: lnet: per NI map-on-demand value James Simmons
` (6 preceding siblings ...)
2016-05-07 1:30 ` [PATCH 07/10] staging: lustre: o2iblnd: per NI map-on-demand value James Simmons
@ 2016-05-07 1:30 ` James Simmons
2016-05-07 1:30 ` [PATCH 09/10] staging: lustre: o2iblnd: properly set ibr_why James Simmons
2016-05-07 1:30 ` [PATCH 10/10] staging: lustre: lnet: enable configuration per NI interface James Simmons
9 siblings, 0 replies; 11+ messages in thread
From: James Simmons @ 2016-05-07 1:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, devel, Andreas Dilger, Oleg Drokin
Cc: Linux Kernel Mailing List, Lustre Development List, Amir Shehata,
James Simmons, James Simmons
From: Amir Shehata <amir.shehata@intel.com>
The function kibnd_tunables_fini() no longer exist. Remove
it from o2iblnd.h
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Signed-off-by: James Simmons <uja.ornl@yahoo.com>
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-7101
Reviewed-on: http://review.whamcloud.com/16367
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Reviewed-by: Olaf Weber <olaf@sgi.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h | 1 -
1 files changed, 0 insertions(+), 1 deletions(-)
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
index d458773..b22984f 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd.h
@@ -995,7 +995,6 @@ void kiblnd_fmr_pool_unmap(kib_fmr_t *fmr, int status);
int kiblnd_tunables_setup(struct lnet_ni *ni);
void kiblnd_tunables_init(void);
-void kiblnd_tunables_fini(void);
int kiblnd_connd(void *arg);
int kiblnd_scheduler(void *arg);
--
1.7.1
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH 09/10] staging: lustre: o2iblnd: properly set ibr_why
2016-05-07 1:30 [PATCH 00/10] staging: lustre: lnet: per NI map-on-demand value James Simmons
` (7 preceding siblings ...)
2016-05-07 1:30 ` [PATCH 08/10] staging: lustre: o2iblnd: remove last of kiblnd_tunables_fini James Simmons
@ 2016-05-07 1:30 ` James Simmons
2016-05-07 1:30 ` [PATCH 10/10] staging: lustre: lnet: enable configuration per NI interface James Simmons
9 siblings, 0 replies; 11+ messages in thread
From: James Simmons @ 2016-05-07 1:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, devel, Andreas Dilger, Oleg Drokin
Cc: Linux Kernel Mailing List, Lustre Development List, Amir Shehata,
James Simmons, James Simmons
From: Amir Shehata <amir.shehata@intel.com>
For the case of when the connections supported fragment count
is smaller than what is supported locally only set ibr_why to
IBLND_REJECT_RDMA_FRAGS if the ko2iblnd protocol verison is
IBLND_MSG_VERSION.
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Signed-off-by: James Simmons <uja.ornl@yahoo.com>
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-7101
Reviewed-on: http://review.whamcloud.com/16367
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Reviewed-by: Olaf Weber <olaf@sgi.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
.../staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
index c4daf95..0f7e3a1 100644
--- a/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/drivers/staging/lustre/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -2368,7 +2368,7 @@ kiblnd_passive_connect(struct rdma_cm_id *cmid, void *priv, int priv_nob)
reqmsg->ibm_u.connparams.ibcp_max_frags,
kiblnd_rdma_frags(version, ni));
- if (version >= IBLND_MSG_VERSION)
+ if (version == IBLND_MSG_VERSION)
rej.ibr_why = IBLND_REJECT_RDMA_FRAGS;
goto failed;
--
1.7.1
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH 10/10] staging: lustre: lnet: enable configuration per NI interface
2016-05-07 1:30 [PATCH 00/10] staging: lustre: lnet: per NI map-on-demand value James Simmons
` (8 preceding siblings ...)
2016-05-07 1:30 ` [PATCH 09/10] staging: lustre: o2iblnd: properly set ibr_why James Simmons
@ 2016-05-07 1:30 ` James Simmons
9 siblings, 0 replies; 11+ messages in thread
From: James Simmons @ 2016-05-07 1:30 UTC (permalink / raw)
To: Greg Kroah-Hartman, devel, Andreas Dilger, Oleg Drokin
Cc: Linux Kernel Mailing List, Lustre Development List, Amir Shehata,
James Simmons, James Simmons
From: Amir Shehata <amir.shehata@intel.com>
Add the ability to configure each NI interface at bring up.
Also give the ability for user land utilities to query the
configuration of each NI interface.
Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Signed-off-by: James Simmons <uja.ornl@yahoo.com>
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-7101
Reviewed-on: http://review.whamcloud.com/16367
Reviewed-by: Doug Oucharek <doug.s.oucharek@intel.com>
Reviewed-by: Olaf Weber <olaf@sgi.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
drivers/staging/lustre/lnet/lnet/api-ni.c | 42 +++++++++++++++++++++++++++++
1 files changed, 42 insertions(+), 0 deletions(-)
diff --git a/drivers/staging/lustre/lnet/lnet/api-ni.c b/drivers/staging/lustre/lnet/lnet/api-ni.c
index 01e1808..fe0dbe7 100644
--- a/drivers/staging/lustre/lnet/lnet/api-ni.c
+++ b/drivers/staging/lustre/lnet/lnet/api-ni.c
@@ -1217,6 +1217,7 @@ lnet_shutdown_lndni(struct lnet_ni *ni)
static int
lnet_startup_lndni(struct lnet_ni *ni, struct lnet_ioctl_config_data *conf)
{
+ struct lnet_ioctl_config_lnd_tunables *lnd_tunables = NULL;
int rc = -EINVAL;
int lnd_type;
lnd_t *lnd;
@@ -1274,6 +1275,21 @@ lnet_startup_lndni(struct lnet_ni *ni, struct lnet_ioctl_config_data *conf)
ni->ni_lnd = lnd;
+ if (conf && conf->cfg_hdr.ioc_len > sizeof(*conf))
+ lnd_tunables = (struct lnet_ioctl_config_lnd_tunables *)conf->cfg_bulk;
+
+ if (lnd_tunables) {
+ LIBCFS_ALLOC(ni->ni_lnd_tunables,
+ sizeof(*ni->ni_lnd_tunables));
+ if (!ni->ni_lnd_tunables) {
+ mutex_unlock(&the_lnet.ln_lnd_mutex);
+ rc = -ENOMEM;
+ goto failed0;
+ }
+ memcpy(ni->ni_lnd_tunables, lnd_tunables,
+ sizeof(*ni->ni_lnd_tunables));
+ }
+
rc = lnd->lnd_startup(ni);
mutex_unlock(&the_lnet.ln_lnd_mutex);
@@ -1653,7 +1669,9 @@ EXPORT_SYMBOL(LNetNIFini);
static void
lnet_fill_ni_info(struct lnet_ni *ni, struct lnet_ioctl_config_data *config)
{
+ struct lnet_ioctl_config_lnd_tunables *lnd_cfg = NULL;
struct lnet_ioctl_net_config *net_config;
+ size_t min_size, tunable_size = 0;
int i;
if (!ni || !config)
@@ -1691,6 +1709,30 @@ lnet_fill_ni_info(struct lnet_ni *ni, struct lnet_ioctl_config_data *config)
config->cfg_ncpts = num_cpts;
}
+
+ /*
+ * See if user land tools sent in a newer and larger version
+ * of struct lnet_tunables than what the kernel uses.
+ */
+ min_size = sizeof(*config) + sizeof(*net_config);
+
+ if (config->cfg_hdr.ioc_len > min_size)
+ tunable_size = config->cfg_hdr.ioc_len - min_size;
+
+ /* Don't copy to much data to user space */
+ min_size = min(tunable_size, sizeof(*ni->ni_lnd_tunables));
+ lnd_cfg = (struct lnet_ioctl_config_lnd_tunables *)net_config->cfg_bulk;
+
+ if (ni->ni_lnd_tunables && lnd_cfg && min_size) {
+ memcpy(lnd_cfg, ni->ni_lnd_tunables, min_size);
+ config->cfg_config_u.cfg_net.net_interface_count = 1;
+
+ /* Tell user land that kernel side has less data */
+ if (tunable_size > sizeof(*ni->ni_lnd_tunables)) {
+ min_size = tunable_size - sizeof(ni->ni_lnd_tunables);
+ config->cfg_hdr.ioc_len -= min_size;
+ }
+ }
}
static int
--
1.7.1
^ permalink raw reply [flat|nested] 11+ messages in thread