From: Roland Dreier <roland@topspin.com>
To: akpm@osdl.org
Cc: linux-kernel@vger.kernel.org, openib-general@openib.org
Subject: [PATCH][9/26] IB/mthca: dynamic context memory mapping for mem-free mode
Date: Thu, 3 Mar 2005 15:20:27 -0800 [thread overview]
Message-ID: <2005331520.nfKPjEcWG6DlwOqo@topspin.com> (raw)
In-Reply-To: <2005331520.qwFp6OBqldRd6oo8@topspin.com>
Add support for mapping more memory into HCA's context to cover
context tables when new objects are allocated. Pass the object
size into mthca_alloc_icm_table(), reference count the ICM chunks,
and add new mthca_table_get() and mthca_table_put() functions to
handle mapping memory when allocating or destroying objects.
Signed-off-by: Roland Dreier <roland@topspin.com>
--- linux-export.orig/drivers/infiniband/hw/mthca/mthca_main.c 2005-03-03 14:12:56.152732681 -0800
+++ linux-export/drivers/infiniband/hw/mthca/mthca_main.c 2005-03-03 14:12:56.772598129 -0800
@@ -363,10 +363,9 @@
}
mdev->mr_table.mtt_table = mthca_alloc_icm_table(mdev, init_hca->mtt_base,
- mdev->limits.num_mtt_segs *
init_hca->mtt_seg_sz,
- mdev->limits.reserved_mtts *
- init_hca->mtt_seg_sz, 1);
+ mdev->limits.num_mtt_segs,
+ mdev->limits.reserved_mtts, 1);
if (!mdev->mr_table.mtt_table) {
mthca_err(mdev, "Failed to map MTT context memory, aborting.\n");
err = -ENOMEM;
@@ -374,10 +373,9 @@
}
mdev->mr_table.mpt_table = mthca_alloc_icm_table(mdev, init_hca->mpt_base,
- mdev->limits.num_mpts *
dev_lim->mpt_entry_sz,
- mdev->limits.reserved_mrws *
- dev_lim->mpt_entry_sz, 1);
+ mdev->limits.num_mpts,
+ mdev->limits.reserved_mrws, 1);
if (!mdev->mr_table.mpt_table) {
mthca_err(mdev, "Failed to map MPT context memory, aborting.\n");
err = -ENOMEM;
@@ -385,10 +383,9 @@
}
mdev->qp_table.qp_table = mthca_alloc_icm_table(mdev, init_hca->qpc_base,
- mdev->limits.num_qps *
dev_lim->qpc_entry_sz,
- mdev->limits.reserved_qps *
- dev_lim->qpc_entry_sz, 1);
+ mdev->limits.num_qps,
+ mdev->limits.reserved_qps, 0);
if (!mdev->qp_table.qp_table) {
mthca_err(mdev, "Failed to map QP context memory, aborting.\n");
err = -ENOMEM;
@@ -396,10 +393,9 @@
}
mdev->qp_table.eqp_table = mthca_alloc_icm_table(mdev, init_hca->eqpc_base,
- mdev->limits.num_qps *
dev_lim->eqpc_entry_sz,
- mdev->limits.reserved_qps *
- dev_lim->eqpc_entry_sz, 1);
+ mdev->limits.num_qps,
+ mdev->limits.reserved_qps, 0);
if (!mdev->qp_table.eqp_table) {
mthca_err(mdev, "Failed to map EQP context memory, aborting.\n");
err = -ENOMEM;
@@ -407,10 +403,9 @@
}
mdev->cq_table.table = mthca_alloc_icm_table(mdev, init_hca->cqc_base,
- mdev->limits.num_cqs *
dev_lim->cqc_entry_sz,
- mdev->limits.reserved_cqs *
- dev_lim->cqc_entry_sz, 1);
+ mdev->limits.num_cqs,
+ mdev->limits.reserved_cqs, 0);
if (!mdev->cq_table.table) {
mthca_err(mdev, "Failed to map CQ context memory, aborting.\n");
err = -ENOMEM;
--- linux-export.orig/drivers/infiniband/hw/mthca/mthca_memfree.c 2005-01-25 20:46:29.000000000 -0800
+++ linux-export/drivers/infiniband/hw/mthca/mthca_memfree.c 2005-03-03 14:12:56.773597912 -0800
@@ -79,6 +79,7 @@
if (!icm)
return icm;
+ icm->refcount = 0;
INIT_LIST_HEAD(&icm->chunk_list);
cur_order = get_order(MTHCA_ICM_ALLOC_SIZE);
@@ -138,9 +139,62 @@
return NULL;
}
+int mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int obj)
+{
+ int i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE;
+ int ret = 0;
+ u8 status;
+
+ down(&table->mutex);
+
+ if (table->icm[i]) {
+ ++table->icm[i]->refcount;
+ goto out;
+ }
+
+ table->icm[i] = mthca_alloc_icm(dev, MTHCA_TABLE_CHUNK_SIZE >> PAGE_SHIFT,
+ (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) |
+ __GFP_NOWARN);
+ if (!table->icm[i]) {
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ if (mthca_MAP_ICM(dev, table->icm[i], table->virt + i * MTHCA_TABLE_CHUNK_SIZE,
+ &status) || status) {
+ mthca_free_icm(dev, table->icm[i]);
+ table->icm[i] = NULL;
+ ret = -ENOMEM;
+ goto out;
+ }
+
+ ++table->icm[i]->refcount;
+
+out:
+ up(&table->mutex);
+ return ret;
+}
+
+void mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int obj)
+{
+ int i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE;
+ u8 status;
+
+ down(&table->mutex);
+
+ if (--table->icm[i]->refcount == 0) {
+ mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE,
+ MTHCA_TABLE_CHUNK_SIZE >> 12, &status);
+ mthca_free_icm(dev, table->icm[i]);
+ table->icm[i] = NULL;
+ }
+
+ up(&table->mutex);
+}
+
struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev,
- u64 virt, unsigned size,
- unsigned reserved,
+ u64 virt, int obj_size,
+ int nobj, int reserved,
int use_lowmem)
{
struct mthca_icm_table *table;
@@ -148,20 +202,23 @@
int i;
u8 status;
- num_icm = size / MTHCA_TABLE_CHUNK_SIZE;
+ num_icm = obj_size * nobj / MTHCA_TABLE_CHUNK_SIZE;
table = kmalloc(sizeof *table + num_icm * sizeof *table->icm, GFP_KERNEL);
if (!table)
return NULL;
- table->virt = virt;
- table->num_icm = num_icm;
- init_MUTEX(&table->sem);
+ table->virt = virt;
+ table->num_icm = num_icm;
+ table->num_obj = nobj;
+ table->obj_size = obj_size;
+ table->lowmem = use_lowmem;
+ init_MUTEX(&table->mutex);
for (i = 0; i < num_icm; ++i)
table->icm[i] = NULL;
- for (i = 0; i < (reserved + MTHCA_TABLE_CHUNK_SIZE - 1) / MTHCA_TABLE_CHUNK_SIZE; ++i) {
+ for (i = 0; i * MTHCA_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) {
table->icm[i] = mthca_alloc_icm(dev, MTHCA_TABLE_CHUNK_SIZE >> PAGE_SHIFT,
(use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) |
__GFP_NOWARN);
@@ -173,6 +230,12 @@
table->icm[i] = NULL;
goto err;
}
+
+ /*
+ * Add a reference to this ICM chunk so that it never
+ * gets freed (since it contains reserved firmware objects).
+ */
+ ++table->icm[i]->refcount;
}
return table;
--- linux-export.orig/drivers/infiniband/hw/mthca/mthca_memfree.h 2005-01-25 20:46:29.000000000 -0800
+++ linux-export/drivers/infiniband/hw/mthca/mthca_memfree.h 2005-03-03 14:12:56.773597912 -0800
@@ -53,12 +53,16 @@
struct mthca_icm {
struct list_head chunk_list;
+ int refcount;
};
struct mthca_icm_table {
u64 virt;
int num_icm;
- struct semaphore sem;
+ int num_obj;
+ int obj_size;
+ int lowmem;
+ struct semaphore mutex;
struct mthca_icm *icm[0];
};
@@ -75,10 +79,12 @@
void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm);
struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev,
- u64 virt, unsigned size,
- unsigned reserved,
+ u64 virt, int obj_size,
+ int nobj, int reserved,
int use_lowmem);
void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table);
+int mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int obj);
+void mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int obj);
static inline void mthca_icm_first(struct mthca_icm *icm,
struct mthca_icm_iter *iter)
next prev parent reply other threads:[~2005-03-04 0:00 UTC|newest]
Thread overview: 47+ messages / expand[flat|nested] mbox.gz Atom feed top
2005-03-03 23:20 [PATCH][0/26] InfiniBand merge Roland Dreier
2005-03-03 23:20 ` [PATCH][1/26] IB: fix ib_find_cached_gid() port numbering Roland Dreier
2005-03-03 23:20 ` [PATCH][2/26] IB/mthca: CQ minor tweaks Roland Dreier
2005-03-03 23:20 ` [PATCH][3/26] IB/mthca: improve CQ locking part 1 Roland Dreier
2005-03-03 23:20 ` [PATCH][4/26] IB/mthca: improve CQ locking part 2 Roland Dreier
2005-03-03 23:20 ` [PATCH][5/26] IB/mthca: CQ cleanups Roland Dreier
2005-03-03 23:20 ` [PATCH][6/26] IB: remove unsignaled receives Roland Dreier
2005-03-03 23:20 ` [PATCH][7/26] IB/mthca: map registers for mem-free mode Roland Dreier
2005-03-03 23:20 ` [PATCH][8/26] IB/mthca: add UAR allocation Roland Dreier
2005-03-03 23:20 ` Roland Dreier [this message]
2005-03-03 23:20 ` [PATCH][10/26] IB/mthca: mem-free memory region support Roland Dreier
2005-03-03 23:20 ` [PATCH][11/26] IB/mthca: mem-free EQ initialization Roland Dreier
2005-03-03 23:20 ` [PATCH][12/26] IB/mthca: mem-free interrupt handling Roland Dreier
2005-03-03 23:20 ` [PATCH][13/26] IB/mthca: tweak firmware command debug messages Roland Dreier
2005-03-03 23:20 ` [PATCH][14/26] IB/mthca: tweak MAP_ICM_page firmware command Roland Dreier
2005-03-03 23:20 ` [PATCH][15/26] IB/mthca: mem-free doorbell record allocation Roland Dreier
2005-03-03 23:20 ` [PATCH][16/26] IB/mthca: mem-free doorbell record writing Roland Dreier
2005-03-03 23:20 ` [PATCH][17/26] IB/mthca: refactor CQ buffer allocate/free Roland Dreier
2005-03-03 23:20 ` [PATCH][18/26] IB/mthca: mem-free CQ initialization Roland Dreier
2005-03-03 23:20 ` [PATCH][19/26] IB/mthca: mem-free CQ operations Roland Dreier
2005-03-03 23:20 ` [PATCH][20/26] IB/mthca: mem-free QP initialization Roland Dreier
2005-03-03 23:20 ` [PATCH][21/26] IB/mthca: mem-free address vectors Roland Dreier
2005-03-03 23:20 ` [PATCH][22/26] IB/mthca: mem-free work request posting Roland Dreier
2005-03-03 23:20 ` [PATCH][23/26] IB/mthca: mem-free multicast table Roland Dreier
2005-03-03 23:20 ` [PATCH][24/26] IB/mthca: QP locking optimization Roland Dreier
2005-03-03 23:20 ` [PATCH][25/26] IB/mthca: implement query of device caps Roland Dreier
2005-03-03 23:20 ` [PATCH][26/26] IB: MAD cancel callbacks from thread Roland Dreier
2005-03-04 0:07 ` Jeff Garzik
2005-03-04 0:30 ` Roland Dreier
2005-03-04 0:34 ` [openib-general] Re: [PATCH][26/26] IB: MAD cancel callbacks fromthread Sean Hefty
2005-03-04 0:43 ` Roland Dreier
2005-03-04 1:01 ` Andrew Morton
2005-03-04 1:07 ` Andrew Morton
2005-03-04 1:22 ` Andrew Morton
2006-03-13 15:43 ` [PATCH 5/6 v2] IB: IP address based RDMA connection manager Roland Dreier
2006-03-13 17:11 ` Sean Hefty
2006-03-13 17:26 ` Roland Dreier
2005-03-04 0:04 ` [PATCH][16/26] IB/mthca: mem-free doorbell record writing Jeff Garzik
2005-03-04 0:33 ` Roland Dreier
2005-03-04 0:41 ` Jeff Garzik
2005-03-04 0:50 ` Roland Dreier
2005-03-04 0:35 ` [PATCH][3/26] IB/mthca: improve CQ locking part 1 Jeff Garzik
2005-03-04 0:40 ` Roland Dreier
2005-03-04 0:58 ` Greg KH
2005-03-04 1:02 ` Roland Dreier
2005-03-04 16:33 ` Greg KH
2005-03-04 16:43 ` Roland Dreier
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=2005331520.nfKPjEcWG6DlwOqo@topspin.com \
--to=roland@topspin.com \
--cc=akpm@osdl.org \
--cc=linux-kernel@vger.kernel.org \
--cc=openib-general@openib.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®