[RFC 00/16] drbd: part 3 of adding multiple volume support to drbd

[RFC 00/16] drbd: part 3 of adding multiple volume support to drbd

[Philipp Reisner]

This the first request for review of drbd-8.4. The complete set has 
492 patches. This is the third installment containing 16 patches.

The whole set is available here:
  git://git.drbd.org/linux-2.6-drbd.git for-jens

and is jens_for-3.2_drivers...for-jens
and this part is ac0f15d...9a88f41

The most noticeable change is the support for multiple replicated volumes in
a single DRBD connection.  Write-ordering is obeyed among all writes in all
volumes in a single connection.  This feature is really important for users
who use DRBD for mirroring over longer distances. (Protocol A).

[PATCH 01/16] drbd: Concurrent write detection fix

[Philipp Reisner]

From: Andreas Gruenbacher <agruen@linbit.com>

Commit 9b1e63e changed the concurrent write detection algorithm to only insert
peer requests into write_requests tree after determining that there is no
conflict.  With this change, new conflicting local requests could be added
while the algorithm runs, but this case was not handled correctly.  Instead of
making the algorithm deal with this case, switch back to adding peer requests
to the write_requests tree immediately: this improves fairness.

When a peer request is discarded, remove that request from the write_requests

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_receiver.c |   41 ++++++++++++++++++-----------------
 1 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index 1b67426..987af1a 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -1639,16 +1639,10 @@ static int e_send_discard_ack(struct drbd_work *w, int unused)
 	struct drbd_peer_request *peer_req =
 		container_of(w, struct drbd_peer_request, w);
 	struct drbd_conf *mdev = w->mdev;
-	int ok = 1;
+	int ok;
 
 	D_ASSERT(mdev->tconn->net_conf->wire_protocol == DRBD_PROT_C);
 	ok = drbd_send_ack(mdev, P_DISCARD_ACK, peer_req);
-
-	spin_lock_irq(&mdev->tconn->req_lock);
-	D_ASSERT(!drbd_interval_empty(&peer_req->i));
-	drbd_remove_epoch_entry_interval(mdev, peer_req);
-	spin_unlock_irq(&mdev->tconn->req_lock);
-
 	dec_unacked(mdev);
 
 	return ok;
@@ -1839,6 +1833,12 @@ static int receive_Data(struct drbd_conf *mdev, enum drbd_packet cmd,
 
 		spin_lock_irq(&mdev->tconn->req_lock);
 
+		/*
+		 * Inserting the peer request into the write_requests tree will
+		 * prevent new conflicting local requests from being added.
+		 */
+		drbd_insert_interval(&mdev->write_requests, &peer_req->i);
+
 		first = 1;
 		for (;;) {
 			struct drbd_interval *i;
@@ -1847,26 +1847,26 @@ static int receive_Data(struct drbd_conf *mdev, enum drbd_packet cmd,
 			prepare_to_wait(&mdev->misc_wait, &wait,
 				TASK_INTERRUPTIBLE);
 
-			i = drbd_find_overlap(&mdev->write_requests, sector, size);
-			if (i) {
+			drbd_for_each_overlap(i, &mdev->write_requests, sector, size) {
+				struct drbd_request *req2;
+
+				if (i == &peer_req->i || !i->local)
+					continue;
+
 				/* only ALERT on first iteration,
 				 * we may be woken up early... */
 				if (first)
-					dev_alert(DEV, "%s[%u] Concurrent %s write detected!"
+					dev_alert(DEV, "%s[%u] Concurrent local write detected!"
 					      "	new: %llus +%u; pending: %llus +%u\n",
 					      current->comm, current->pid,
-					      i->local ? "local" : "remote",
 					      (unsigned long long)sector, size,
 					      (unsigned long long)i->sector, i->size);
 
-				if (i->local) {
-					struct drbd_request *req2;
-
-					req2 = container_of(i, struct drbd_request, i);
-					if (req2->rq_state & RQ_NET_PENDING)
-						++have_unacked;
-				}
+				req2 = container_of(i, struct drbd_request, i);
+				if (req2->rq_state & RQ_NET_PENDING)
+					++have_unacked;
 				++have_conflict;
+				break;
 			}
 			if (!have_conflict)
 				break;
@@ -1875,6 +1875,7 @@ static int receive_Data(struct drbd_conf *mdev, enum drbd_packet cmd,
 			if (first && discard && have_unacked) {
 				dev_alert(DEV, "Concurrent write! [DISCARD BY FLAG] sec=%llus\n",
 				     (unsigned long long)sector);
+				drbd_remove_epoch_entry_interval(mdev, peer_req);
 				inc_unacked(mdev);
 				peer_req->w.cb = e_send_discard_ack;
 				list_add_tail(&peer_req->w.list, &mdev->done_ee);
@@ -1891,6 +1892,7 @@ static int receive_Data(struct drbd_conf *mdev, enum drbd_packet cmd,
 			}
 
 			if (signal_pending(current)) {
+				drbd_remove_epoch_entry_interval(mdev, peer_req);
 				spin_unlock_irq(&mdev->tconn->req_lock);
 				finish_wait(&mdev->misc_wait, &wait);
 				goto out_interrupted;
@@ -1909,12 +1911,11 @@ static int receive_Data(struct drbd_conf *mdev, enum drbd_packet cmd,
 				 * there must be none now. */
 				D_ASSERT(have_unacked == 0);
 			}
+			/* FIXME: Introduce a timeout here after which we disconnect.  */
 			schedule();
 			spin_lock_irq(&mdev->tconn->req_lock);
 		}
 		finish_wait(&mdev->misc_wait, &wait);
-
-		drbd_insert_interval(&mdev->write_requests, &peer_req->i);
 	}
 
 	list_add(&peer_req->w.list, &mdev->active_ee);
-- 
1.7.4.1

[PATCH 02/16] drbd: Replace atomic_add_return with atomic_inc_return

[Philipp Reisner]

From: Andreas Gruenbacher <agruen@linbit.com>

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_main.c |    4 ++--
 drivers/block/drbd/drbd_nl.c   |    8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index 47c4bae..e9e60d8 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -1242,7 +1242,7 @@ static int _drbd_send_ack(struct drbd_conf *mdev, enum drbd_packet cmd,
 	p.sector   = sector;
 	p.block_id = block_id;
 	p.blksize  = blksize;
-	p.seq_num  = cpu_to_be32(atomic_add_return(1, &mdev->packet_seq));
+	p.seq_num  = cpu_to_be32(atomic_inc_return(&mdev->packet_seq));
 
 	if (!mdev->tconn->meta.socket || mdev->state.conn < C_CONNECTED)
 		return false;
@@ -1530,7 +1530,7 @@ int drbd_send_dblock(struct drbd_conf *mdev, struct drbd_request *req)
 	prepare_header(mdev, &p.head, P_DATA, sizeof(p) - sizeof(struct p_header) + dgs + req->i.size);
 	p.sector   = cpu_to_be64(req->i.sector);
 	p.block_id = (unsigned long)req;
-	p.seq_num  = cpu_to_be32(req->seq_num = atomic_add_return(1, &mdev->packet_seq));
+	p.seq_num  = cpu_to_be32(req->seq_num = atomic_inc_return(&mdev->packet_seq));
 
 	dp_flags = bio_flags_to_wire(mdev, req->master_bio->bi_rw);
 
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index 90d7317..0168587 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -2374,7 +2374,7 @@ void drbd_bcast_state(struct drbd_conf *mdev, union drbd_state state)
 	cn_reply->id.idx = CN_IDX_DRBD;
 	cn_reply->id.val = CN_VAL_DRBD;
 
-	cn_reply->seq = atomic_add_return(1, &drbd_nl_seq);
+	cn_reply->seq = atomic_inc_return(&drbd_nl_seq);
 	cn_reply->ack = 0; /* not used here. */
 	cn_reply->len = sizeof(struct drbd_nl_cfg_reply) +
 		(int)((char *)tl - (char *)reply->tag_list);
@@ -2406,7 +2406,7 @@ void drbd_bcast_ev_helper(struct drbd_conf *mdev, char *helper_name)
 	cn_reply->id.idx = CN_IDX_DRBD;
 	cn_reply->id.val = CN_VAL_DRBD;
 
-	cn_reply->seq = atomic_add_return(1, &drbd_nl_seq);
+	cn_reply->seq = atomic_inc_return(&drbd_nl_seq);
 	cn_reply->ack = 0; /* not used here. */
 	cn_reply->len = sizeof(struct drbd_nl_cfg_reply) +
 		(int)((char *)tl - (char *)reply->tag_list);
@@ -2485,7 +2485,7 @@ void drbd_bcast_ee(struct drbd_conf *mdev, const char *reason, const int dgs,
 	cn_reply->id.idx = CN_IDX_DRBD;
 	cn_reply->id.val = CN_VAL_DRBD;
 
-	cn_reply->seq = atomic_add_return(1,&drbd_nl_seq);
+	cn_reply->seq = atomic_inc_return(&drbd_nl_seq);
 	cn_reply->ack = 0; // not used here.
 	cn_reply->len = sizeof(struct drbd_nl_cfg_reply) +
 		(int)((char*)tl - (char*)reply->tag_list);
@@ -2524,7 +2524,7 @@ void drbd_bcast_sync_progress(struct drbd_conf *mdev)
 	cn_reply->id.idx = CN_IDX_DRBD;
 	cn_reply->id.val = CN_VAL_DRBD;
 
-	cn_reply->seq = atomic_add_return(1, &drbd_nl_seq);
+	cn_reply->seq = atomic_inc_return(&drbd_nl_seq);
 	cn_reply->ack = 0; /* not used here. */
 	cn_reply->len = sizeof(struct drbd_nl_cfg_reply) +
 		(int)((char *)tl - (char *)reply->tag_list);
-- 
1.7.4.1

[PATCH 03/16] drbd: Use ping-timeout when waiting for missing ack packets

[Philipp Reisner]

From: Andreas Gruenbacher <agruen@linbit.com>

When the node with the discard flag resolves write conflicts in
dual-primary mode, it may determine that its peer has sent ack packets
on the metadata socket which did not arrive, yet.  Wait for the next ack
with ping-timeout instead of a hard-coded 30 seconds.

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_receiver.c |    5 +++--
 1 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index 987af1a..07fc7c1 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -1713,11 +1713,12 @@ static int drbd_wait_peer_seq(struct drbd_conf *mdev, const u32 packet_seq)
 		}
 		p_seq = mdev->peer_seq;
 		spin_unlock(&mdev->peer_seq_lock);
-		timeout = schedule_timeout(30*HZ);
+		timeout = mdev->tconn->net_conf->ping_timeo*HZ/10;
+		timeout = schedule_timeout(timeout);
 		spin_lock(&mdev->peer_seq_lock);
 		if (timeout == 0 && p_seq == mdev->peer_seq) {
 			ret = -ETIMEDOUT;
-			dev_err(DEV, "ASSERT FAILED waited 30 seconds for sequence update, forcing reconnect\n");
+			dev_err(DEV, "Timed out waiting for missing ack packets; disconnecting\n");
 			break;
 		}
 	}
-- 
1.7.4.1

[PATCH 04/16] drbd: Improve how conflicting writes are handled

[Philipp Reisner]

From: Andreas Gruenbacher <agruen@linbit.com>

The previous algorithm for dealing with overlapping concurrent writes
was generating unnecessary warnings for scenarios which could be
legitimate, and did not always handle partially overlapping requests
correctly.  Improve it algorithm as follows:

* While local or remote write requests are in progress, conflicting new
  local write requests will be delayed (commit 82172f7).

* When a conflict between a local and remote write request is detected,
  the node with the discard flag decides how to resolve the conflict: It
  will ask its peer to discard conflicting requests which are fully
  contained in the local request and retry requests which overlap only
  partially.  This involves a protocol change.

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_int.h      |   15 +-
 drivers/block/drbd/drbd_main.c     |   35 +++-
 drivers/block/drbd/drbd_receiver.c |  410 ++++++++++++++++++++++--------------
 drivers/block/drbd/drbd_req.c      |   75 ++++---
 drivers/block/drbd/drbd_req.h      |    7 +-
 5 files changed, 351 insertions(+), 191 deletions(-)

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index 482bd7e..cda1039 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -200,7 +200,7 @@ enum drbd_packet {
 	P_RECV_ACK	      = 0x15, /* Used in protocol B */
 	P_WRITE_ACK	      = 0x16, /* Used in protocol C */
 	P_RS_WRITE_ACK	      = 0x17, /* Is a P_WRITE_ACK, additionally call set_in_sync(). */
-	P_DISCARD_ACK	      = 0x18, /* Used in proto C, two-primaries conflict detection */
+	P_DISCARD_WRITE	      = 0x18, /* Used in proto C, two-primaries conflict detection */
 	P_NEG_ACK	      = 0x19, /* Sent if local disk is unusable */
 	P_NEG_DREPLY	      = 0x1a, /* Local disk is broken... */
 	P_NEG_RS_DREPLY	      = 0x1b, /* Local disk is broken... */
@@ -223,8 +223,9 @@ enum drbd_packet {
 	P_RS_CANCEL           = 0x29, /* meta: Used to cancel RS_DATA_REQUEST packet by SyncSource */
 	P_CONN_ST_CHG_REQ     = 0x2a, /* data sock: Connection wide state request */
 	P_CONN_ST_CHG_REPLY   = 0x2b, /* meta sock: Connection side state req reply */
+	P_RETRY_WRITE	      = 0x2c, /* Protocol C: retry conflicting write request */
 
-	P_MAX_CMD	      = 0x2c,
+	P_MAX_CMD	      = 0x2d,
 	P_MAY_IGNORE	      = 0x100, /* Flag to test if (cmd > P_MAY_IGNORE) ... */
 	P_MAX_OPT_CMD	      = 0x101,
 
@@ -350,7 +351,7 @@ struct p_data {
  * commands which share a struct:
  *  p_block_ack:
  *   P_RECV_ACK (proto B), P_WRITE_ACK (proto C),
- *   P_DISCARD_ACK (proto C, two-primaries conflict detection)
+ *   P_DISCARD_WRITE (proto C, two-primaries conflict detection)
  *  p_block_req:
  *   P_DATA_REQUEST, P_RS_DATA_REQUEST
  */
@@ -362,7 +363,6 @@ struct p_block_ack {
 	u32	    seq_num;
 } __packed;
 
-
 struct p_block_req {
 	struct p_header head;
 	u64 sector;
@@ -655,6 +655,8 @@ struct drbd_work {
 
 #include "drbd_interval.h"
 
+extern int drbd_wait_misc(struct drbd_conf *, struct drbd_interval *);
+
 struct drbd_request {
 	struct drbd_work w;
 
@@ -752,12 +754,16 @@ enum {
 
 	/* This ee has a pointer to a digest instead of a block id */
 	__EE_HAS_DIGEST,
+
+	/* Conflicting local requests need to be restarted after this request */
+	__EE_RESTART_REQUESTS,
 };
 #define EE_CALL_AL_COMPLETE_IO (1<<__EE_CALL_AL_COMPLETE_IO)
 #define EE_MAY_SET_IN_SYNC     (1<<__EE_MAY_SET_IN_SYNC)
 #define	EE_RESUBMITTED         (1<<__EE_RESUBMITTED)
 #define EE_WAS_ERROR           (1<<__EE_WAS_ERROR)
 #define EE_HAS_DIGEST          (1<<__EE_HAS_DIGEST)
+#define EE_RESTART_REQUESTS	(1<<__EE_RESTART_REQUESTS)
 
 /* flag bits per mdev */
 enum {
@@ -1478,6 +1484,7 @@ extern void drbd_free_tconn(struct drbd_tconn *tconn);
 extern int proc_details;
 
 /* drbd_req */
+extern int __drbd_make_request(struct drbd_conf *, struct bio *, unsigned long);
 extern int drbd_make_request(struct request_queue *q, struct bio *bio);
 extern int drbd_read_remote(struct drbd_conf *mdev, struct drbd_request *req);
 extern int drbd_merge_bvec(struct request_queue *q, struct bvec_merge_data *bvm, struct bio_vec *bvec);
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index e9e60d8..e71b6bf 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -3004,7 +3004,7 @@ const char *cmdname(enum drbd_packet cmd)
 		[P_RECV_ACK]	        = "RecvAck",
 		[P_WRITE_ACK]	        = "WriteAck",
 		[P_RS_WRITE_ACK]	= "RSWriteAck",
-		[P_DISCARD_ACK]	        = "DiscardAck",
+		[P_DISCARD_WRITE]        = "DiscardWrite",
 		[P_NEG_ACK]	        = "NegAck",
 		[P_NEG_DREPLY]	        = "NegDReply",
 		[P_NEG_RS_DREPLY]	= "NegRSDReply",
@@ -3019,6 +3019,7 @@ const char *cmdname(enum drbd_packet cmd)
 		[P_COMPRESSED_BITMAP]   = "CBitmap",
 		[P_DELAY_PROBE]         = "DelayProbe",
 		[P_OUT_OF_SYNC]		= "OutOfSync",
+		[P_RETRY_WRITE]		= "RetryWrite",
 		[P_MAX_CMD]	        = NULL,
 	};
 
@@ -3033,6 +3034,38 @@ const char *cmdname(enum drbd_packet cmd)
 	return cmdnames[cmd];
 }
 
+/**
+ * drbd_wait_misc  -  wait for a request to make progress
+ * @mdev:	device associated with the request
+ * @i:		the struct drbd_interval embedded in struct drbd_request or
+ *		struct drbd_peer_request
+ */
+int drbd_wait_misc(struct drbd_conf *mdev, struct drbd_interval *i)
+{
+	struct net_conf *net_conf = mdev->tconn->net_conf;
+	DEFINE_WAIT(wait);
+	long timeout;
+
+	if (!net_conf)
+		return -ETIMEDOUT;
+	timeout = MAX_SCHEDULE_TIMEOUT;
+	if (net_conf->ko_count)
+		timeout = net_conf->timeout * HZ / 10 * net_conf->ko_count;
+
+	/* Indicate to wake up mdev->misc_wait on progress.  */
+	i->waiting = true;
+	prepare_to_wait(&mdev->misc_wait, &wait, TASK_INTERRUPTIBLE);
+	spin_unlock_irq(&mdev->tconn->req_lock);
+	timeout = schedule_timeout(timeout);
+	finish_wait(&mdev->misc_wait, &wait);
+	spin_lock_irq(&mdev->tconn->req_lock);
+	if (!timeout || mdev->state.conn < C_CONNECTED)
+		return -ETIMEDOUT;
+	if (signal_pending(current))
+		return -ERESTARTSYS;
+	return 0;
+}
+
 #ifdef CONFIG_DRBD_FAULT_INJECTION
 /* Fault insertion support including random number generator shamelessly
  * stolen from kernel/rcutorture.c */
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index 07fc7c1..6d19b57 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -415,7 +415,7 @@ static int drbd_process_done_ee(struct drbd_conf *mdev)
 		drbd_free_net_ee(mdev, peer_req);
 
 	/* possible callbacks here:
-	 * e_end_block, and e_end_resync_block, e_send_discard_ack.
+	 * e_end_block, and e_end_resync_block, e_send_discard_write.
 	 * all ignore the last argument.
 	 */
 	list_for_each_entry_safe(peer_req, t, &work_list, w.list) {
@@ -1592,6 +1592,51 @@ static int receive_RSDataReply(struct drbd_conf *mdev, enum drbd_packet cmd,
 	return ok;
 }
 
+static int w_restart_write(struct drbd_work *w, int cancel)
+{
+	struct drbd_request *req = container_of(w, struct drbd_request, w);
+	struct drbd_conf *mdev = w->mdev;
+	struct bio *bio;
+	unsigned long start_time;
+	unsigned long flags;
+
+	spin_lock_irqsave(&mdev->tconn->req_lock, flags);
+	if (!expect(req->rq_state & RQ_POSTPONED)) {
+		spin_unlock_irqrestore(&mdev->tconn->req_lock, flags);
+		return 0;
+	}
+	bio = req->master_bio;
+	start_time = req->start_time;
+	/* Postponed requests will not have their master_bio completed!  */
+	__req_mod(req, DISCARD_WRITE, NULL);
+	spin_unlock_irqrestore(&mdev->tconn->req_lock, flags);
+
+	while (__drbd_make_request(mdev, bio, start_time))
+		/* retry */ ;
+	return 1;
+}
+
+static void restart_conflicting_writes(struct drbd_conf *mdev,
+				       sector_t sector, int size)
+{
+	struct drbd_interval *i;
+	struct drbd_request *req;
+
+	drbd_for_each_overlap(i, &mdev->write_requests, sector, size) {
+		if (!i->local)
+			continue;
+		req = container_of(i, struct drbd_request, i);
+		if (req->rq_state & RQ_LOCAL_PENDING ||
+		    !(req->rq_state & RQ_POSTPONED))
+			continue;
+		if (expect(list_empty(&req->w.list))) {
+			req->w.mdev = mdev;
+			req->w.cb = w_restart_write;
+			drbd_queue_work(&mdev->tconn->data.work, &req->w);
+		}
+	}
+}
+
 /* e_end_block() is called via drbd_process_done_ee().
  * this means this function only runs in the asender thread
  */
@@ -1625,6 +1670,8 @@ static int e_end_block(struct drbd_work *w, int cancel)
 		spin_lock_irq(&mdev->tconn->req_lock);
 		D_ASSERT(!drbd_interval_empty(&peer_req->i));
 		drbd_remove_epoch_entry_interval(mdev, peer_req);
+		if (peer_req->flags & EE_RESTART_REQUESTS)
+			restart_conflicting_writes(mdev, sector, peer_req->i.size);
 		spin_unlock_irq(&mdev->tconn->req_lock);
 	} else
 		D_ASSERT(drbd_interval_empty(&peer_req->i));
@@ -1634,20 +1681,32 @@ static int e_end_block(struct drbd_work *w, int cancel)
 	return ok;
 }
 
-static int e_send_discard_ack(struct drbd_work *w, int unused)
+static int e_send_ack(struct drbd_work *w, enum drbd_packet ack)
 {
+	struct drbd_conf *mdev = w->mdev;
 	struct drbd_peer_request *peer_req =
 		container_of(w, struct drbd_peer_request, w);
-	struct drbd_conf *mdev = w->mdev;
 	int ok;
 
-	D_ASSERT(mdev->tconn->net_conf->wire_protocol == DRBD_PROT_C);
-	ok = drbd_send_ack(mdev, P_DISCARD_ACK, peer_req);
+	ok = drbd_send_ack(mdev, ack, peer_req);
 	dec_unacked(mdev);
 
 	return ok;
 }
 
+static int e_send_discard_write(struct drbd_work *w, int unused)
+{
+	return e_send_ack(w, P_DISCARD_WRITE);
+}
+
+static int e_send_retry_write(struct drbd_work *w, int unused)
+{
+	struct drbd_tconn *tconn = w->mdev->tconn;
+
+	return e_send_ack(w, tconn->agreed_pro_version >= 100 ?
+			     P_RETRY_WRITE : P_DISCARD_WRITE);
+}
+
 static bool seq_greater(u32 a, u32 b)
 {
 	/*
@@ -1663,16 +1722,31 @@ static u32 seq_max(u32 a, u32 b)
 	return seq_greater(a, b) ? a : b;
 }
 
+static bool need_peer_seq(struct drbd_conf *mdev)
+{
+	struct drbd_tconn *tconn = mdev->tconn;
+
+	/*
+	 * We only need to keep track of the last packet_seq number of our peer
+	 * if we are in dual-primary mode and we have the discard flag set; see
+	 * handle_write_conflicts().
+	 */
+	return tconn->net_conf->two_primaries &&
+	       test_bit(DISCARD_CONCURRENT, &tconn->flags);
+}
+
 static void update_peer_seq(struct drbd_conf *mdev, unsigned int peer_seq)
 {
 	unsigned int old_peer_seq;
 
-	spin_lock(&mdev->peer_seq_lock);
-	old_peer_seq = mdev->peer_seq;
-	mdev->peer_seq = seq_max(mdev->peer_seq, peer_seq);
-	spin_unlock(&mdev->peer_seq_lock);
-	if (old_peer_seq != peer_seq)
-		wake_up(&mdev->seq_wait);
+	if (need_peer_seq(mdev)) {
+		spin_lock(&mdev->peer_seq_lock);
+		old_peer_seq = mdev->peer_seq;
+		mdev->peer_seq = seq_max(mdev->peer_seq, peer_seq);
+		spin_unlock(&mdev->peer_seq_lock);
+		if (old_peer_seq != peer_seq)
+			wake_up(&mdev->seq_wait);
+	}
 }
 
 /* Called from receive_Data.
@@ -1696,36 +1770,39 @@ static void update_peer_seq(struct drbd_conf *mdev, unsigned int peer_seq)
  *
  * returns 0 if we may process the packet,
  * -ERESTARTSYS if we were interrupted (by disconnect signal). */
-static int drbd_wait_peer_seq(struct drbd_conf *mdev, const u32 packet_seq)
+static int wait_for_and_update_peer_seq(struct drbd_conf *mdev, const u32 peer_seq)
 {
 	DEFINE_WAIT(wait);
-	unsigned int p_seq;
 	long timeout;
-	int ret = 0;
+	int ret;
+
+	if (!need_peer_seq(mdev))
+		return 0;
+
 	spin_lock(&mdev->peer_seq_lock);
 	for (;;) {
-		prepare_to_wait(&mdev->seq_wait, &wait, TASK_INTERRUPTIBLE);
-		if (!seq_greater(packet_seq, mdev->peer_seq + 1))
+		if (!seq_greater(peer_seq - 1, mdev->peer_seq)) {
+			mdev->peer_seq = seq_max(mdev->peer_seq, peer_seq);
+			ret = 0;
 			break;
+		}
 		if (signal_pending(current)) {
 			ret = -ERESTARTSYS;
 			break;
 		}
-		p_seq = mdev->peer_seq;
+		prepare_to_wait(&mdev->seq_wait, &wait, TASK_INTERRUPTIBLE);
 		spin_unlock(&mdev->peer_seq_lock);
 		timeout = mdev->tconn->net_conf->ping_timeo*HZ/10;
 		timeout = schedule_timeout(timeout);
 		spin_lock(&mdev->peer_seq_lock);
-		if (timeout == 0 && p_seq == mdev->peer_seq) {
+		if (!timeout) {
 			ret = -ETIMEDOUT;
 			dev_err(DEV, "Timed out waiting for missing ack packets; disconnecting\n");
 			break;
 		}
 	}
-	finish_wait(&mdev->seq_wait, &wait);
-	if (mdev->peer_seq+1 == packet_seq)
-		mdev->peer_seq++;
 	spin_unlock(&mdev->peer_seq_lock);
+	finish_wait(&mdev->seq_wait, &wait);
 	return ret;
 }
 
@@ -1740,6 +1817,139 @@ static unsigned long wire_flags_to_bio(struct drbd_conf *mdev, u32 dpf)
 		(dpf & DP_DISCARD ? REQ_DISCARD : 0);
 }
 
+static void fail_postponed_requests(struct drbd_conf *mdev, sector_t sector,
+				    unsigned int size)
+{
+	struct drbd_interval *i;
+
+    repeat:
+	drbd_for_each_overlap(i, &mdev->write_requests, sector, size) {
+		struct drbd_request *req;
+		struct bio_and_error m;
+
+		if (!i->local)
+			continue;
+		req = container_of(i, struct drbd_request, i);
+		if (!(req->rq_state & RQ_POSTPONED))
+			continue;
+		req->rq_state &= ~RQ_POSTPONED;
+		__req_mod(req, NEG_ACKED, &m);
+		spin_unlock_irq(&mdev->tconn->req_lock);
+		if (m.bio)
+			complete_master_bio(mdev, &m);
+		spin_lock_irq(&mdev->tconn->req_lock);
+		goto repeat;
+	}
+}
+
+static int handle_write_conflicts(struct drbd_conf *mdev,
+				  struct drbd_peer_request *peer_req)
+{
+	struct drbd_tconn *tconn = mdev->tconn;
+	bool resolve_conflicts = test_bit(DISCARD_CONCURRENT, &tconn->flags);
+	sector_t sector = peer_req->i.sector;
+	const unsigned int size = peer_req->i.size;
+	struct drbd_interval *i;
+	bool equal;
+	int err;
+
+	/*
+	 * Inserting the peer request into the write_requests tree will prevent
+	 * new conflicting local requests from being added.
+	 */
+	drbd_insert_interval(&mdev->write_requests, &peer_req->i);
+
+    repeat:
+	drbd_for_each_overlap(i, &mdev->write_requests, sector, size) {
+		if (i == &peer_req->i)
+			continue;
+
+		if (!i->local) {
+			/*
+			 * Our peer has sent a conflicting remote request; this
+			 * should not happen in a two-node setup.  Wait for the
+			 * earlier peer request to complete.
+			 */
+			err = drbd_wait_misc(mdev, i);
+			if (err)
+				goto out;
+			goto repeat;
+		}
+
+		equal = i->sector == sector && i->size == size;
+		if (resolve_conflicts) {
+			/*
+			 * If the peer request is fully contained within the
+			 * overlapping request, it can be discarded; otherwise,
+			 * it will be retried once all overlapping requests
+			 * have completed.
+			 */
+			bool discard = i->sector <= sector && i->sector +
+				       (i->size >> 9) >= sector + (size >> 9);
+
+			if (!equal)
+				dev_alert(DEV, "Concurrent writes detected: "
+					       "local=%llus +%u, remote=%llus +%u, "
+					       "assuming %s came first\n",
+					  (unsigned long long)i->sector, i->size,
+					  (unsigned long long)sector, size,
+					  discard ? "local" : "remote");
+
+			inc_unacked(mdev);
+			peer_req->w.cb = discard ? e_send_discard_write :
+						   e_send_retry_write;
+			list_add_tail(&peer_req->w.list, &mdev->done_ee);
+			wake_asender(mdev->tconn);
+
+			err = -ENOENT;
+			goto out;
+		} else {
+			struct drbd_request *req =
+				container_of(i, struct drbd_request, i);
+
+			if (!equal)
+				dev_alert(DEV, "Concurrent writes detected: "
+					       "local=%llus +%u, remote=%llus +%u\n",
+					  (unsigned long long)i->sector, i->size,
+					  (unsigned long long)sector, size);
+
+			if (req->rq_state & RQ_LOCAL_PENDING ||
+			    !(req->rq_state & RQ_POSTPONED)) {
+				/*
+				 * Wait for the node with the discard flag to
+				 * decide if this request will be discarded or
+				 * retried.  Requests that are discarded will
+				 * disappear from the write_requests tree.
+				 *
+				 * In addition, wait for the conflicting
+				 * request to finish locally before submitting
+				 * the conflicting peer request.
+				 */
+				err = drbd_wait_misc(mdev, &req->i);
+				if (err) {
+					_conn_request_state(mdev->tconn,
+							    NS(conn, C_TIMEOUT),
+							    CS_HARD);
+					fail_postponed_requests(mdev, sector, size);
+					goto out;
+				}
+				goto repeat;
+			}
+			/*
+			 * Remember to restart the conflicting requests after
+			 * the new peer request has completed.
+			 */
+			peer_req->flags |= EE_RESTART_REQUESTS;
+		}
+	}
+	err = 0;
+
+    out:
+	if (err)
+		drbd_remove_epoch_entry_interval(mdev, peer_req);
+	return err;
+}
+
 /* mirrored write */
 static int receive_Data(struct drbd_conf *mdev, enum drbd_packet cmd,
 			unsigned int data_size)
@@ -1747,18 +1957,17 @@ static int receive_Data(struct drbd_conf *mdev, enum drbd_packet cmd,
 	sector_t sector;
 	struct drbd_peer_request *peer_req;
 	struct p_data *p = &mdev->tconn->data.rbuf.data;
+	u32 peer_seq = be32_to_cpu(p->seq_num);
 	int rw = WRITE;
 	u32 dp_flags;
+	int err;
 
-	if (!get_ldev(mdev)) {
-		spin_lock(&mdev->peer_seq_lock);
-		if (mdev->peer_seq+1 == be32_to_cpu(p->seq_num))
-			mdev->peer_seq++;
-		spin_unlock(&mdev->peer_seq_lock);
 
+	if (!get_ldev(mdev)) {
+		err = wait_for_and_update_peer_seq(mdev, peer_seq);
 		drbd_send_ack_dp(mdev, P_NEG_ACK, p, data_size);
 		atomic_inc(&mdev->current_epoch->epoch_size);
-		return drbd_drain_block(mdev, data_size);
+		return drbd_drain_block(mdev, data_size) && err == 0;
 	}
 
 	/*
@@ -1788,137 +1997,22 @@ static int receive_Data(struct drbd_conf *mdev, enum drbd_packet cmd,
 	atomic_inc(&peer_req->epoch->active);
 	spin_unlock(&mdev->epoch_lock);
 
-	/* I'm the receiver, I do hold a net_cnt reference. */
-	if (!mdev->tconn->net_conf->two_primaries) {
-		spin_lock_irq(&mdev->tconn->req_lock);
-	} else {
-		/* don't get the req_lock yet,
-		 * we may sleep in drbd_wait_peer_seq */
-		const int size = peer_req->i.size;
-		const int discard = test_bit(DISCARD_CONCURRENT, &mdev->tconn->flags);
-		DEFINE_WAIT(wait);
-		int first;
-
-		D_ASSERT(mdev->tconn->net_conf->wire_protocol == DRBD_PROT_C);
-
-		/* conflict detection and handling:
-		 * 1. wait on the sequence number,
-		 *    in case this data packet overtook ACK packets.
-		 * 2. check for conflicting write requests.
-		 *
-		 * Note: for two_primaries, we are protocol C,
-		 * so there cannot be any request that is DONE
-		 * but still on the transfer log.
-		 *
-		 * if no conflicting request is found:
-		 *    submit.
-		 *
-		 * if any conflicting request is found
-		 * that has not yet been acked,
-		 * AND I have the "discard concurrent writes" flag:
-		 *	 queue (via done_ee) the P_DISCARD_ACK; OUT.
-		 *
-		 * if any conflicting request is found:
-		 *	 block the receiver, waiting on misc_wait
-		 *	 until no more conflicting requests are there,
-		 *	 or we get interrupted (disconnect).
-		 *
-		 *	 we do not just write after local io completion of those
-		 *	 requests, but only after req is done completely, i.e.
-		 *	 we wait for the P_DISCARD_ACK to arrive!
-		 *
-		 *	 then proceed normally, i.e. submit.
-		 */
-		if (drbd_wait_peer_seq(mdev, be32_to_cpu(p->seq_num)))
+	if (mdev->tconn->net_conf->two_primaries) {
+		err = wait_for_and_update_peer_seq(mdev, peer_seq);
+		if (err)
 			goto out_interrupted;
-
 		spin_lock_irq(&mdev->tconn->req_lock);
-
-		/*
-		 * Inserting the peer request into the write_requests tree will
-		 * prevent new conflicting local requests from being added.
-		 */
-		drbd_insert_interval(&mdev->write_requests, &peer_req->i);
-
-		first = 1;
-		for (;;) {
-			struct drbd_interval *i;
-			int have_unacked = 0;
-			int have_conflict = 0;
-			prepare_to_wait(&mdev->misc_wait, &wait,
-				TASK_INTERRUPTIBLE);
-
-			drbd_for_each_overlap(i, &mdev->write_requests, sector, size) {
-				struct drbd_request *req2;
-
-				if (i == &peer_req->i || !i->local)
-					continue;
-
-				/* only ALERT on first iteration,
-				 * we may be woken up early... */
-				if (first)
-					dev_alert(DEV, "%s[%u] Concurrent local write detected!"
-					      "	new: %llus +%u; pending: %llus +%u\n",
-					      current->comm, current->pid,
-					      (unsigned long long)sector, size,
-					      (unsigned long long)i->sector, i->size);
-
-				req2 = container_of(i, struct drbd_request, i);
-				if (req2->rq_state & RQ_NET_PENDING)
-					++have_unacked;
-				++have_conflict;
-				break;
-			}
-			if (!have_conflict)
-				break;
-
-			/* Discard Ack only for the _first_ iteration */
-			if (first && discard && have_unacked) {
-				dev_alert(DEV, "Concurrent write! [DISCARD BY FLAG] sec=%llus\n",
-				     (unsigned long long)sector);
-				drbd_remove_epoch_entry_interval(mdev, peer_req);
-				inc_unacked(mdev);
-				peer_req->w.cb = e_send_discard_ack;
-				list_add_tail(&peer_req->w.list, &mdev->done_ee);
-
-				spin_unlock_irq(&mdev->tconn->req_lock);
-
-				/* we could probably send that P_DISCARD_ACK ourselves,
-				 * but I don't like the receiver using the msock */
-
+		err = handle_write_conflicts(mdev, peer_req);
+		if (err) {
+			spin_unlock_irq(&mdev->tconn->req_lock);
+			if (err == -ENOENT) {
 				put_ldev(mdev);
-				wake_asender(mdev->tconn);
-				finish_wait(&mdev->misc_wait, &wait);
 				return true;
 			}
-
-			if (signal_pending(current)) {
-				drbd_remove_epoch_entry_interval(mdev, peer_req);
-				spin_unlock_irq(&mdev->tconn->req_lock);
-				finish_wait(&mdev->misc_wait, &wait);
-				goto out_interrupted;
-			}
-
-			/* Indicate to wake up mdev->misc_wait upon completion.  */
-			i->waiting = true;
-
-			spin_unlock_irq(&mdev->tconn->req_lock);
-			if (first) {
-				first = 0;
-				dev_alert(DEV, "Concurrent write! [W AFTERWARDS] "
-				     "sec=%llus\n", (unsigned long long)sector);
-			} else if (discard) {
-				/* we had none on the first iteration.
-				 * there must be none now. */
-				D_ASSERT(have_unacked == 0);
-			}
-			/* FIXME: Introduce a timeout here after which we disconnect.  */
-			schedule();
-			spin_lock_irq(&mdev->tconn->req_lock);
+			goto out_interrupted;
 		}
-		finish_wait(&mdev->misc_wait, &wait);
-	}
-
+	} else
+		spin_lock_irq(&mdev->tconn->req_lock);
 	list_add(&peer_req->w.list, &mdev->active_ee);
 	spin_unlock_irq(&mdev->tconn->req_lock);
 
@@ -4396,9 +4490,13 @@ static int got_BlockAck(struct drbd_conf *mdev, enum drbd_packet cmd)
 		D_ASSERT(mdev->tconn->net_conf->wire_protocol == DRBD_PROT_B);
 		what = RECV_ACKED_BY_PEER;
 		break;
-	case P_DISCARD_ACK:
+	case P_DISCARD_WRITE:
 		D_ASSERT(mdev->tconn->net_conf->wire_protocol == DRBD_PROT_C);
-		what = CONFLICT_DISCARDED_BY_PEER;
+		what = DISCARD_WRITE;
+		break;
+	case P_RETRY_WRITE:
+		D_ASSERT(mdev->tconn->net_conf->wire_protocol == DRBD_PROT_C);
+		what = POSTPONE_WRITE;
 		break;
 	default:
 		D_ASSERT(0);
@@ -4449,6 +4547,7 @@ static int got_NegDReply(struct drbd_conf *mdev, enum drbd_packet cmd)
 	sector_t sector = be64_to_cpu(p->sector);
 
 	update_peer_seq(mdev, be32_to_cpu(p->seq_num));
+
 	dev_err(DEV, "Got NegDReply; Sector %llus, len %u; Fail original request.\n",
 	    (unsigned long long)sector, be32_to_cpu(p->blksize));
 
@@ -4570,7 +4669,7 @@ static struct asender_cmd *get_asender_cmd(int cmd)
 	[P_RECV_ACK]	    = { sizeof(struct p_block_ack), got_BlockAck },
 	[P_WRITE_ACK]	    = { sizeof(struct p_block_ack), got_BlockAck },
 	[P_RS_WRITE_ACK]    = { sizeof(struct p_block_ack), got_BlockAck },
-	[P_DISCARD_ACK]	    = { sizeof(struct p_block_ack), got_BlockAck },
+	[P_DISCARD_WRITE]   = { sizeof(struct p_block_ack), got_BlockAck },
 	[P_NEG_ACK]	    = { sizeof(struct p_block_ack), got_NegAck },
 	[P_NEG_DREPLY]	    = { sizeof(struct p_block_ack), got_NegDReply },
 	[P_NEG_RS_DREPLY]   = { sizeof(struct p_block_ack), got_NegRSDReply},
@@ -4581,6 +4680,7 @@ static struct asender_cmd *get_asender_cmd(int cmd)
 	[P_DELAY_PROBE]     = { sizeof(struct p_delay_probe93), got_skip },
 	[P_RS_CANCEL]       = { sizeof(struct p_block_ack), got_NegRSDReply},
 	[P_CONN_ST_CHG_REPLY]={ sizeof(struct p_req_state_reply), got_RqSReply },
+	[P_RETRY_WRITE]	    = { sizeof(struct p_block_ack), got_BlockAck },
 	[P_MAX_CMD]	    = { 0, NULL },
 	};
 	if (cmd > P_MAX_CMD || asender_tbl[cmd].process == NULL)
diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c
index 7332198..aab1acd 100644
--- a/drivers/block/drbd/drbd_req.c
+++ b/drivers/block/drbd/drbd_req.c
@@ -225,12 +225,16 @@ void _req_may_be_done(struct drbd_request *req, struct bio_and_error *m)
 	 *	the receiver,
 	 *	the bio_endio completion callbacks.
 	 */
+	if (s & RQ_LOCAL_PENDING)
+		return;
+	if (req->i.waiting) {
+		/* Retry all conflicting peer requests.  */
+		wake_up(&mdev->misc_wait);
+	}
 	if (s & RQ_NET_QUEUED)
 		return;
 	if (s & RQ_NET_PENDING)
 		return;
-	if (s & RQ_LOCAL_PENDING)
-		return;
 
 	if (req->master_bio) {
 		/* this is DATA_RECEIVED (remote read)
@@ -267,7 +271,7 @@ void _req_may_be_done(struct drbd_request *req, struct bio_and_error *m)
 			else
 				root = &mdev->read_requests;
 			drbd_remove_request_interval(root, req);
-		} else
+		} else if (!(s & RQ_POSTPONED))
 			D_ASSERT((s & (RQ_NET_MASK & ~RQ_NET_DONE)) == 0);
 
 		/* for writes we need to do some extra housekeeping */
@@ -277,8 +281,10 @@ void _req_may_be_done(struct drbd_request *req, struct bio_and_error *m)
 		/* Update disk stats */
 		_drbd_end_io_acct(mdev, req);
 
-		m->error = ok ? 0 : (error ?: -EIO);
-		m->bio = req->master_bio;
+		if (!(s & RQ_POSTPONED)) {
+			m->error = ok ? 0 : (error ?: -EIO);
+			m->bio = req->master_bio;
+		}
 		req->master_bio = NULL;
 	}
 
@@ -318,7 +324,9 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what,
 {
 	struct drbd_conf *mdev = req->w.mdev;
 	int rv = 0;
-	m->bio = NULL;
+
+	if (m)
+		m->bio = NULL;
 
 	switch (what) {
 	default:
@@ -332,7 +340,7 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what,
 		*/
 
 	case TO_BE_SENT: /* via network */
-		/* reached via drbd_make_request_common
+		/* reached via __drbd_make_request
 		 * and from w_read_retry_remote */
 		D_ASSERT(!(req->rq_state & RQ_NET_MASK));
 		req->rq_state |= RQ_NET_PENDING;
@@ -340,7 +348,7 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what,
 		break;
 
 	case TO_BE_SUBMITTED: /* locally */
-		/* reached via drbd_make_request_common */
+		/* reached via __drbd_make_request */
 		D_ASSERT(!(req->rq_state & RQ_LOCAL_MASK));
 		req->rq_state |= RQ_LOCAL_PENDING;
 		break;
@@ -403,7 +411,7 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what,
 		 * no local disk,
 		 * or target area marked as invalid,
 		 * or just got an io-error. */
-		/* from drbd_make_request_common
+		/* from __drbd_make_request
 		 * or from bio_endio during read io-error recovery */
 
 		/* so we can verify the handle in the answer packet
@@ -422,7 +430,7 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what,
 
 	case QUEUE_FOR_NET_WRITE:
 		/* assert something? */
-		/* from drbd_make_request_common only */
+		/* from __drbd_make_request only */
 
 		/* corresponding hlist_del is in _req_may_be_done() */
 		drbd_insert_interval(&mdev->write_requests, &req->i);
@@ -436,7 +444,7 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what,
 		 *
 		 * _req_add_to_epoch(req); this has to be after the
 		 * _maybe_start_new_epoch(req); which happened in
-		 * drbd_make_request_common, because we now may set the bit
+		 * __drbd_make_request, because we now may set the bit
 		 * again ourselves to close the current epoch.
 		 *
 		 * Add req to the (now) current epoch (barrier). */
@@ -446,7 +454,7 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what,
 		 * hurting performance. */
 		set_bit(UNPLUG_REMOTE, &mdev->flags);
 
-		/* see drbd_make_request_common,
+		/* see __drbd_make_request,
 		 * just after it grabs the req_lock */
 		D_ASSERT(test_bit(CREATE_BARRIER, &mdev->flags) == 0);
 
@@ -535,14 +543,10 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what,
 
 	case WRITE_ACKED_BY_PEER_AND_SIS:
 		req->rq_state |= RQ_NET_SIS;
-	case CONFLICT_DISCARDED_BY_PEER:
+	case DISCARD_WRITE:
 		/* for discarded conflicting writes of multiple primaries,
 		 * there is no need to keep anything in the tl, potential
 		 * node crashes are covered by the activity log. */
-		if (what == CONFLICT_DISCARDED_BY_PEER)
-			dev_alert(DEV, "Got DiscardAck packet %llus +%u!"
-			      " DRBD is not a random data generator!\n",
-			      (unsigned long long)req->i.sector, req->i.size);
 		req->rq_state |= RQ_NET_DONE;
 		/* fall through */
 	case WRITE_ACKED_BY_PEER:
@@ -569,6 +573,17 @@ int __req_mod(struct drbd_request *req, enum drbd_req_event what,
 		_req_may_be_done_not_susp(req, m);
 		break;
 
+	case POSTPONE_WRITE:
+		/*
+		 * If this node has already detected the write conflict, the
+		 * worker will be waiting on misc_wait.  Wake it up once this
+		 * request has completed locally.
+		 */
+		D_ASSERT(req->rq_state & RQ_NET_PENDING);
+		req->rq_state |= RQ_POSTPONED;
+		_req_may_be_done_not_susp(req, m);
+		break;
+
 	case NEG_ACKED:
 		/* assert something? */
 		if (req->rq_state & RQ_NET_PENDING) {
@@ -688,24 +703,19 @@ static int complete_conflicting_writes(struct drbd_conf *mdev,
 				       sector_t sector, int size)
 {
 	for(;;) {
-		DEFINE_WAIT(wait);
 		struct drbd_interval *i;
+		int err;
 
 		i = drbd_find_overlap(&mdev->write_requests, sector, size);
 		if (!i)
 			return 0;
-		i->waiting = true;
-		prepare_to_wait(&mdev->misc_wait, &wait, TASK_INTERRUPTIBLE);
-		spin_unlock_irq(&mdev->tconn->req_lock);
-		schedule();
-		finish_wait(&mdev->misc_wait, &wait);
-		spin_lock_irq(&mdev->tconn->req_lock);
-		if (signal_pending(current))
-			return -ERESTARTSYS;
+		err = drbd_wait_misc(mdev, i);
+		if (err)
+			return err;
 	}
 }
 
-static int drbd_make_request_common(struct drbd_conf *mdev, struct bio *bio, unsigned long start_time)
+int __drbd_make_request(struct drbd_conf *mdev, struct bio *bio, unsigned long start_time)
 {
 	const int rw = bio_rw(bio);
 	const int size = bio->bi_size;
@@ -811,7 +821,12 @@ allocate_barrier:
 	if (rw == WRITE) {
 		err = complete_conflicting_writes(mdev, sector, size);
 		if (err) {
+			if (err != -ERESTARTSYS)
+				_conn_request_state(mdev->tconn,
+						    NS(conn, C_TIMEOUT),
+						    CS_HARD);
 			spin_unlock_irq(&mdev->tconn->req_lock);
+			err = -EIO;
 			goto fail_free_complete;
 		}
 	}
@@ -1031,7 +1046,7 @@ int drbd_make_request(struct request_queue *q, struct bio *bio)
 
 	if (likely(s_enr == e_enr)) {
 		inc_ap_bio(mdev, 1);
-		return drbd_make_request_common(mdev, bio, start_time);
+		return __drbd_make_request(mdev, bio, start_time);
 	}
 
 	/* can this bio be split generically?
@@ -1069,10 +1084,10 @@ int drbd_make_request(struct request_queue *q, struct bio *bio)
 
 		D_ASSERT(e_enr == s_enr + 1);
 
-		while (drbd_make_request_common(mdev, &bp->bio1, start_time))
+		while (__drbd_make_request(mdev, &bp->bio1, start_time))
 			inc_ap_bio(mdev, 1);
 
-		while (drbd_make_request_common(mdev, &bp->bio2, start_time))
+		while (__drbd_make_request(mdev, &bp->bio2, start_time))
 			inc_ap_bio(mdev, 1);
 
 		dec_ap_bio(mdev);
diff --git a/drivers/block/drbd/drbd_req.h b/drivers/block/drbd/drbd_req.h
index e6f2361..0b3cd41 100644
--- a/drivers/block/drbd/drbd_req.h
+++ b/drivers/block/drbd/drbd_req.h
@@ -97,7 +97,8 @@ enum drbd_req_event {
 	RECV_ACKED_BY_PEER,
 	WRITE_ACKED_BY_PEER,
 	WRITE_ACKED_BY_PEER_AND_SIS, /* and set_in_sync */
-	CONFLICT_DISCARDED_BY_PEER,
+	DISCARD_WRITE,
+	POSTPONE_WRITE,
 	NEG_ACKED,
 	BARRIER_ACKED, /* in protocol A and B */
 	DATA_RECEIVED, /* (remote read) */
@@ -194,6 +195,9 @@ enum drbd_req_state_bits {
 
 	/* Should call drbd_al_complete_io() for this request... */
 	__RQ_IN_ACT_LOG,
+
+	/* The peer has sent a retry ACK */
+	__RQ_POSTPONED,
 };
 
 #define RQ_LOCAL_PENDING   (1UL << __RQ_LOCAL_PENDING)
@@ -214,6 +218,7 @@ enum drbd_req_state_bits {
 
 #define RQ_WRITE           (1UL << __RQ_WRITE)
 #define RQ_IN_ACT_LOG      (1UL << __RQ_IN_ACT_LOG)
+#define RQ_POSTPONED	   (1UL << __RQ_POSTPONED)
 
 /* For waking up the frozen transfer log mod_req() has to return if the request
    should be counted in the epoch object*/
-- 
1.7.4.1

[PATCH 05/16] drbd: Remove redundant check

[Philipp Reisner]

From: Andreas Gruenbacher <agruen@linbit.com>

Opening a device only succeeds on a primary node, or when explicitly
setting the allow_oos module parameter to allow opening the device
read-only on a secondary node.  There is no other way that a request can
get into drbd_make_request(), so this code cannot trigger.

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_req.c |   27 ---------------------------
 1 files changed, 0 insertions(+), 27 deletions(-)

diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c
index aab1acd..cfa5fba 100644
--- a/drivers/block/drbd/drbd_req.c
+++ b/drivers/block/drbd/drbd_req.c
@@ -997,39 +997,12 @@ fail_and_free_req:
 	return ret;
 }
 
-/* helper function for drbd_make_request
- * if we can determine just by the mdev (state) that this request will fail,
- * return 1
- * otherwise return 0
- */
-static int drbd_fail_request_early(struct drbd_conf *mdev, int is_write)
-{
-	if (mdev->state.role != R_PRIMARY &&
-		(!allow_oos || is_write)) {
-		if (__ratelimit(&drbd_ratelimit_state)) {
-			dev_err(DEV, "Process %s[%u] tried to %s; "
-			    "since we are not in Primary state, "
-			    "we cannot allow this\n",
-			    current->comm, current->pid,
-			    is_write ? "WRITE" : "READ");
-		}
-		return 1;
-	}
-
-	return 0;
-}
-
 int drbd_make_request(struct request_queue *q, struct bio *bio)
 {
 	unsigned int s_enr, e_enr;
 	struct drbd_conf *mdev = (struct drbd_conf *) q->queuedata;
 	unsigned long start_time;
 
-	if (drbd_fail_request_early(mdev, bio_data_dir(bio) & WRITE)) {
-		bio_endio(bio, -EPERM);
-		return 0;
-	}
-
 	start_time = jiffies;
 
 	/*
-- 
1.7.4.1

[PATCH 06/16] drbd: Get rid of P_MAX_CMD

[Philipp Reisner]

From: Andreas Gruenbacher <agruen@linbit.com>

Instead of artificially enlarging the command decoding arrays to
P_MAX_CMD entries, check if an index is within the valid range using the
ARRAY_SIZE() macro.

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_int.h      |    1 -
 drivers/block/drbd/drbd_main.c     |    3 +--
 drivers/block/drbd/drbd_receiver.c |   10 ++++------
 3 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index cda1039..c7b9fc9 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -225,7 +225,6 @@ enum drbd_packet {
 	P_CONN_ST_CHG_REPLY   = 0x2b, /* meta sock: Connection side state req reply */
 	P_RETRY_WRITE	      = 0x2c, /* Protocol C: retry conflicting write request */
 
-	P_MAX_CMD	      = 0x2d,
 	P_MAY_IGNORE	      = 0x100, /* Flag to test if (cmd > P_MAY_IGNORE) ... */
 	P_MAX_OPT_CMD	      = 0x101,
 
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index e71b6bf..eb51650 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -3020,7 +3020,6 @@ const char *cmdname(enum drbd_packet cmd)
 		[P_DELAY_PROBE]         = "DelayProbe",
 		[P_OUT_OF_SYNC]		= "OutOfSync",
 		[P_RETRY_WRITE]		= "RetryWrite",
-		[P_MAX_CMD]	        = NULL,
 	};
 
 	if (cmd == P_HAND_SHAKE_M)
@@ -3029,7 +3028,7 @@ const char *cmdname(enum drbd_packet cmd)
 		return "HandShakeS";
 	if (cmd == P_HAND_SHAKE)
 		return "HandShake";
-	if (cmd >= P_MAX_CMD)
+	if (cmd >= ARRAY_SIZE(cmdnames))
 		return "Unknown";
 	return cmdnames[cmd];
 }
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index 6d19b57..9b29348 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -3878,9 +3878,6 @@ static struct data_cmd drbd_cmd_handler[] = {
 	[P_DELAY_PROBE]     = { 0, sizeof(struct p_delay_probe93), receive_skip },
 	[P_OUT_OF_SYNC]     = { 0, sizeof(struct p_block_desc), receive_out_of_sync },
 	[P_CONN_ST_CHG_REQ] = { 0, sizeof(struct p_req_state), receive_req_state },
-	/* anything missing from this table is in
-	 * the asender_tbl, see get_asender_cmd */
-	[P_MAX_CMD]	    = { 0, 0, NULL },
 };
 
 /* All handler functions that expect a sub-header get that sub-heder in
@@ -3902,7 +3899,8 @@ static void drbdd(struct drbd_tconn *tconn)
 		if (!drbd_recv_header(tconn, &pi))
 			goto err_out;
 
-		if (unlikely(pi.cmd >= P_MAX_CMD || !drbd_cmd_handler[pi.cmd].function)) {
+		if (unlikely(pi.cmd >= ARRAY_SIZE(drbd_cmd_handler) ||
+		    !drbd_cmd_handler[pi.cmd].function)) {
 			conn_err(tconn, "unknown packet type %d, l: %d!\n", pi.cmd, pi.size);
 			goto err_out;
 		}
@@ -4681,9 +4679,9 @@ static struct asender_cmd *get_asender_cmd(int cmd)
 	[P_RS_CANCEL]       = { sizeof(struct p_block_ack), got_NegRSDReply},
 	[P_CONN_ST_CHG_REPLY]={ sizeof(struct p_req_state_reply), got_RqSReply },
 	[P_RETRY_WRITE]	    = { sizeof(struct p_block_ack), got_BlockAck },
-	[P_MAX_CMD]	    = { 0, NULL },
 	};
-	if (cmd > P_MAX_CMD || asender_tbl[cmd].process == NULL)
+
+	if (cmd >= ARRAY_SIZE(asender_tbl) || !asender_tbl[cmd].process)
 		return NULL;
 	return &asender_tbl[cmd];
 }
-- 
1.7.4.1

[PATCH 07/16] drbd: Replace get_asender_cmd() with its implementation

[Philipp Reisner]

From: Andreas Gruenbacher <agruen@linbit.com>

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_receiver.c |   66 +++++++++++++++--------------------
 1 files changed, 28 insertions(+), 38 deletions(-)

diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index 9b29348..d78ab6e 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -4651,41 +4651,6 @@ static int got_skip(struct drbd_conf *mdev, enum drbd_packet cmd)
 	return true;
 }
 
-struct asender_cmd {
-	size_t pkt_size;
-	int (*process)(struct drbd_conf *mdev, enum drbd_packet cmd);
-};
-
-static struct asender_cmd *get_asender_cmd(int cmd)
-{
-	static struct asender_cmd asender_tbl[] = {
-		/* anything missing from this table is in
-		 * the drbd_cmd_handler (drbd_default_handler) table,
-		 * see the beginning of drbdd() */
-	[P_PING]	    = { sizeof(struct p_header), got_Ping },
-	[P_PING_ACK]	    = { sizeof(struct p_header), got_PingAck },
-	[P_RECV_ACK]	    = { sizeof(struct p_block_ack), got_BlockAck },
-	[P_WRITE_ACK]	    = { sizeof(struct p_block_ack), got_BlockAck },
-	[P_RS_WRITE_ACK]    = { sizeof(struct p_block_ack), got_BlockAck },
-	[P_DISCARD_WRITE]   = { sizeof(struct p_block_ack), got_BlockAck },
-	[P_NEG_ACK]	    = { sizeof(struct p_block_ack), got_NegAck },
-	[P_NEG_DREPLY]	    = { sizeof(struct p_block_ack), got_NegDReply },
-	[P_NEG_RS_DREPLY]   = { sizeof(struct p_block_ack), got_NegRSDReply},
-	[P_OV_RESULT]	    = { sizeof(struct p_block_ack), got_OVResult },
-	[P_BARRIER_ACK]	    = { sizeof(struct p_barrier_ack), got_BarrierAck },
-	[P_STATE_CHG_REPLY] = { sizeof(struct p_req_state_reply), got_RqSReply },
-	[P_RS_IS_IN_SYNC]   = { sizeof(struct p_block_ack), got_IsInSync },
-	[P_DELAY_PROBE]     = { sizeof(struct p_delay_probe93), got_skip },
-	[P_RS_CANCEL]       = { sizeof(struct p_block_ack), got_NegRSDReply},
-	[P_CONN_ST_CHG_REPLY]={ sizeof(struct p_req_state_reply), got_RqSReply },
-	[P_RETRY_WRITE]	    = { sizeof(struct p_block_ack), got_BlockAck },
-	};
-
-	if (cmd >= ARRAY_SIZE(asender_tbl) || !asender_tbl[cmd].process)
-		return NULL;
-	return &asender_tbl[cmd];
-}
-
 static int _drbd_process_done_ee(int vnr, void *p, void *data)
 {
 	struct drbd_conf *mdev = (struct drbd_conf *)p;
@@ -4722,6 +4687,31 @@ static int tconn_process_done_ee(struct drbd_tconn *tconn)
 	return 0;
 }
 
+struct asender_cmd {
+	size_t pkt_size;
+	int (*process)(struct drbd_conf *, enum drbd_packet);
+};
+
+static struct asender_cmd asender_tbl[] = {
+	[P_PING]	    = { sizeof(struct p_header), got_Ping },
+	[P_PING_ACK]	    = { sizeof(struct p_header), got_PingAck },
+	[P_RECV_ACK]	    = { sizeof(struct p_block_ack), got_BlockAck },
+	[P_WRITE_ACK]	    = { sizeof(struct p_block_ack), got_BlockAck },
+	[P_RS_WRITE_ACK]    = { sizeof(struct p_block_ack), got_BlockAck },
+	[P_DISCARD_WRITE]   = { sizeof(struct p_block_ack), got_BlockAck },
+	[P_NEG_ACK]	    = { sizeof(struct p_block_ack), got_NegAck },
+	[P_NEG_DREPLY]	    = { sizeof(struct p_block_ack), got_NegDReply },
+	[P_NEG_RS_DREPLY]   = { sizeof(struct p_block_ack), got_NegRSDReply},
+	[P_OV_RESULT]	    = { sizeof(struct p_block_ack), got_OVResult },
+	[P_BARRIER_ACK]	    = { sizeof(struct p_barrier_ack), got_BarrierAck },
+	[P_STATE_CHG_REPLY] = { sizeof(struct p_req_state_reply), got_RqSReply },
+	[P_RS_IS_IN_SYNC]   = { sizeof(struct p_block_ack), got_IsInSync },
+	[P_DELAY_PROBE]     = { sizeof(struct p_delay_probe93), got_skip },
+	[P_RS_CANCEL]       = { sizeof(struct p_block_ack), got_NegRSDReply},
+	[P_CONN_ST_CHG_REPLY]={ sizeof(struct p_req_state_reply), got_RqSReply },
+	[P_RETRY_WRITE]	    = { sizeof(struct p_block_ack), got_BlockAck },
+};
+
 int drbd_asender(struct drbd_thread *thi)
 {
 	struct drbd_tconn *tconn = thi->tconn;
@@ -4806,8 +4796,8 @@ int drbd_asender(struct drbd_thread *thi)
 		if (received == expect && cmd == NULL) {
 			if (!decode_header(tconn, h, &pi))
 				goto reconnect;
-			cmd = get_asender_cmd(pi.cmd);
-			if (unlikely(cmd == NULL)) {
+			cmd = &asender_tbl[pi.cmd];
+			if (pi.cmd >= ARRAY_SIZE(asender_tbl) || !cmd) {
 				conn_err(tconn, "unknown command %d on meta (l: %d)\n",
 					pi.cmd, pi.size);
 				goto disconnect;
@@ -4826,7 +4816,7 @@ int drbd_asender(struct drbd_thread *thi)
 
 			/* the idle_timeout (ping-int)
 			 * has been restored in got_PingAck() */
-			if (cmd == get_asender_cmd(P_PING_ACK))
+			if (cmd == &asender_tbl[P_PING_ACK])
 				ping_timeout_active = 0;
 
 			buf	 = h;
-- 
1.7.4.1

[PATCH 08/16] drbd: Remove left-over function prototypes

[Philipp Reisner]

From: Andreas Gruenbacher <agruen@linbit.com>

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_int.h      |    2 --
 drivers/block/drbd/drbd_receiver.c |    1 -
 2 files changed, 0 insertions(+), 3 deletions(-)

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index c7b9fc9..e065985 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -1534,7 +1534,6 @@ extern void drbd_csum_bio(struct drbd_conf *, struct crypto_hash *, struct bio *
 extern void drbd_csum_ee(struct drbd_conf *, struct crypto_hash *,
 			 struct drbd_peer_request *, void *);
 /* worker callbacks */
-extern int w_req_cancel_conflict(struct drbd_work *, int);
 extern int w_read_retry_remote(struct drbd_work *, int);
 extern int w_e_end_data_req(struct drbd_work *, int);
 extern int w_e_end_rsdata_req(struct drbd_work *, int);
@@ -1543,7 +1542,6 @@ extern int w_e_end_ov_reply(struct drbd_work *, int);
 extern int w_e_end_ov_req(struct drbd_work *, int);
 extern int w_ov_finished(struct drbd_work *, int);
 extern int w_resync_timer(struct drbd_work *, int);
-extern int w_resume_next_sg(struct drbd_work *, int);
 extern int w_send_write_hint(struct drbd_work *, int);
 extern int w_send_dblock(struct drbd_work *, int);
 extern int w_send_barrier(struct drbd_work *, int);
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index d78ab6e..88552e1 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -4012,7 +4012,6 @@ static int drbd_disconnected(int vnr, void *p, void *data)
 
 	del_timer(&mdev->request_timer);
 
-	/* make sure syncer is stopped and w_resume_next_sg queued */
 	del_timer_sync(&mdev->resync_timer);
 	resync_timer_fn((unsigned long)mdev);
 
-- 
1.7.4.1

[PATCH 09/16] drbd: Reworked the unconfiguring and thread stopping code

[Philipp Reisner]

* Moved CONFIG_PENDING and DEVICE_DYING from mdev to tconn.
* Renamed drbd_reconfig_start() and drbd_reconfig_done() to
  conn_reconfig_start() and conn_reconfig_done().

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_int.h      |   18 +++++++----
 drivers/block/drbd/drbd_nl.c       |   40 ++++++++++++--------------
 drivers/block/drbd/drbd_receiver.c |    6 ++--
 drivers/block/drbd/drbd_state.c    |   54 ++++++++++++++++++++++-------------
 drivers/block/drbd/drbd_state.h    |    1 +
 drivers/block/drbd/drbd_worker.c   |   31 ++++++++------------
 6 files changed, 80 insertions(+), 70 deletions(-)

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index e065985..d53fb0f 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -787,12 +787,6 @@ enum {
 	GO_DISKLESS,		/* Disk is being detached, on io-error or admin request. */
 	WAS_IO_ERROR,		/* Local disk failed returned IO error */
 	RESYNC_AFTER_NEG,       /* Resync after online grow after the attach&negotiate finished. */
-	CONFIG_PENDING,		/* serialization of (re)configuration requests.
-				 * if set, also prevents the device from dying */
-	DEVICE_DYING,		/* device became unconfigured,
-				 * but worker thread is still handling the cleanup.
-				 * reconfiguring (nl_disk_conf, nl_net_conf) is dissalowed,
-				 * while this is set. */
 	RESIZE_PENDING,		/* Size change detected locally, waiting for the response from
 				 * the peer, if it changed there as well. */
 	CONN_DRY_RUN,		/* Expect disconnect after resync handshake. */
@@ -921,6 +915,12 @@ enum {
 	GOT_PING_ACK,		/* set when we receive a ping_ack packet, ping_wait gets woken */
 	CONN_WD_ST_CHG_OKAY,
 	CONN_WD_ST_CHG_FAIL,
+	CONFIG_PENDING,		/* serialization of (re)configuration requests.
+				 * if set, also prevents the device from dying */
+	OBJECT_DYING,		/* device became unconfigured,
+				 * but worker thread is still handling the cleanup.
+				 * reconfiguring (nl_disk_conf, nl_net_conf) is dissalowed,
+				 * while this is set. */
 };
 
 struct drbd_tconn {			/* is a resource from the config file */
@@ -1574,7 +1574,11 @@ extern void _drbd_wait_ee_list_empty(struct drbd_conf *mdev,
 		struct list_head *head);
 extern void drbd_set_recv_tcq(struct drbd_conf *mdev, int tcq_enabled);
 extern void _drbd_clear_done_ee(struct drbd_conf *mdev, struct list_head *to_be_freed);
-extern void drbd_flush_workqueue(struct drbd_conf *mdev);
+extern void conn_flush_workqueue(struct drbd_tconn *tconn);
+static inline void drbd_flush_workqueue(struct drbd_conf *mdev)
+{
+	conn_flush_workqueue(mdev->tconn);
+}
 
 /* yes, there is kernel_setsockopt, but only since 2.6.18. we don't need to
  * mess with get_fs/set_fs, we know we are KERNEL_DS always. */
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index 0168587..8cdfb46 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -871,29 +871,27 @@ void drbd_reconsider_max_bio_size(struct drbd_conf *mdev)
  * or start a new one.  Flush any pending work, there may still be an
  * after_state_change queued.
  */
-static void drbd_reconfig_start(struct drbd_conf *mdev)
+static void conn_reconfig_start(struct drbd_tconn *tconn)
 {
-	wait_event(mdev->state_wait, !test_and_set_bit(CONFIG_PENDING, &mdev->flags));
-	wait_event(mdev->state_wait, !test_bit(DEVICE_DYING, &mdev->flags));
-	drbd_thread_start(&mdev->tconn->worker);
-	drbd_flush_workqueue(mdev);
+	wait_event(tconn->ping_wait, !test_and_set_bit(CONFIG_PENDING, &tconn->flags));
+	wait_event(tconn->ping_wait, !test_bit(OBJECT_DYING, &tconn->flags));
+	drbd_thread_start(&tconn->worker);
+	conn_flush_workqueue(tconn);
 }
 
 /* if still unconfigured, stops worker again.
  * if configured now, clears CONFIG_PENDING.
  * wakes potential waiters */
-static void drbd_reconfig_done(struct drbd_conf *mdev)
+static void conn_reconfig_done(struct drbd_tconn *tconn)
 {
-	spin_lock_irq(&mdev->tconn->req_lock);
-	if (mdev->state.disk == D_DISKLESS &&
-	    mdev->state.conn == C_STANDALONE &&
-	    mdev->state.role == R_SECONDARY) {
-		set_bit(DEVICE_DYING, &mdev->flags);
-		drbd_thread_stop_nowait(&mdev->tconn->worker);
+	spin_lock_irq(&tconn->req_lock);
+	if (conn_all_vols_unconf(tconn)) {
+		set_bit(OBJECT_DYING, &tconn->flags);
+		drbd_thread_stop_nowait(&tconn->worker);
 	} else
-		clear_bit(CONFIG_PENDING, &mdev->flags);
-	spin_unlock_irq(&mdev->tconn->req_lock);
-	wake_up(&mdev->state_wait);
+		clear_bit(CONFIG_PENDING, &tconn->flags);
+	spin_unlock_irq(&tconn->req_lock);
+	wake_up(&tconn->ping_wait);
 }
 
 /* Make sure IO is suspended before calling this function(). */
@@ -933,7 +931,7 @@ static int drbd_nl_disk_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp
 	enum drbd_state_rv rv;
 	int cp_discovered = 0;
 
-	drbd_reconfig_start(mdev);
+	conn_reconfig_start(mdev->tconn);
 
 	/* if you want to reconfigure, please tear down first */
 	if (mdev->state.disk > D_DISKLESS) {
@@ -1279,7 +1277,7 @@ static int drbd_nl_disk_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp
 	kobject_uevent(&disk_to_dev(mdev->vdisk)->kobj, KOBJ_CHANGE);
 	put_ldev(mdev);
 	reply->ret_code = retcode;
-	drbd_reconfig_done(mdev);
+	conn_reconfig_done(mdev->tconn);
 	return 0;
 
  force_diskless_dec:
@@ -1300,7 +1298,7 @@ static int drbd_nl_disk_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp
 	lc_destroy(resync_lru);
 
 	reply->ret_code = retcode;
-	drbd_reconfig_done(mdev);
+	conn_reconfig_done(mdev->tconn);
 	return 0;
 }
 
@@ -1344,7 +1342,7 @@ static int drbd_nl_net_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
 	void *int_dig_vv = NULL;
 	struct sockaddr *new_my_addr, *new_peer_addr, *taken_addr;
 
-	drbd_reconfig_start(mdev);
+	conn_reconfig_start(mdev->tconn);
 
 	if (mdev->state.conn > C_STANDALONE) {
 		retcode = ERR_NET_CONFIGURED;
@@ -1530,7 +1528,7 @@ static int drbd_nl_net_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
 
 	kobject_uevent(&disk_to_dev(mdev->vdisk)->kobj, KOBJ_CHANGE);
 	reply->ret_code = retcode;
-	drbd_reconfig_done(mdev);
+	conn_reconfig_done(mdev->tconn);
 	return 0;
 
 fail:
@@ -1543,7 +1541,7 @@ fail:
 	kfree(new_conf);
 
 	reply->ret_code = retcode;
-	drbd_reconfig_done(mdev);
+	conn_reconfig_done(mdev->tconn);
 	return 0;
 }
 
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index 88552e1..b58dc16 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -3935,14 +3935,14 @@ static void drbdd(struct drbd_tconn *tconn)
 	}
 }
 
-void drbd_flush_workqueue(struct drbd_conf *mdev)
+void conn_flush_workqueue(struct drbd_tconn *tconn)
 {
 	struct drbd_wq_barrier barr;
 
 	barr.w.cb = w_prev_work_done;
-	barr.w.mdev = mdev;
+	barr.w.tconn = tconn;
 	init_completion(&barr.done);
-	drbd_queue_work(&mdev->tconn->data.work, &barr.w);
+	drbd_queue_work(&tconn->data.work, &barr.w);
 	wait_for_completion(&barr.done);
 }
 
diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index 61c65f5..febefcc 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -41,13 +41,29 @@ extern void _tl_restart(struct drbd_conf *mdev, enum drbd_req_event what);
 static int w_after_state_ch(struct drbd_work *w, int unused);
 static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
 			   union drbd_state ns, enum chg_state_flags flags);
-static void after_all_state_ch(struct drbd_tconn *tconn, union drbd_state ns);
+static void after_all_state_ch(struct drbd_tconn *tconn);
 static enum drbd_state_rv is_valid_state(struct drbd_conf *, union drbd_state);
 static enum drbd_state_rv is_valid_soft_transition(union drbd_state, union drbd_state);
 static enum drbd_state_rv is_valid_transition(union drbd_state os, union drbd_state ns);
 static union drbd_state sanitize_state(struct drbd_conf *mdev, union drbd_state ns,
 				       const char **warn_sync_abort);
 
+int conn_all_vols_unconf(struct drbd_tconn *tconn)
+{
+	struct drbd_conf *mdev;
+	int minor, uncfg = 1;
+
+	idr_for_each_entry(&tconn->volumes, mdev, minor) {
+		uncfg &= (mdev->state.disk == D_DISKLESS &&
+			  mdev->state.conn == C_STANDALONE &&
+			  mdev->state.role == R_SECONDARY);
+		if (!uncfg)
+			break;
+	}
+
+	return uncfg;
+}
+
 /**
  * cl_wide_st_chg() - true if the state change is a cluster wide one
  * @mdev:	DRBD device.
@@ -744,20 +760,6 @@ __drbd_set_state(struct drbd_conf *mdev, union drbd_state ns,
 
 	print_state_change(mdev, os, ns, flags);
 
-	/* solve the race between becoming unconfigured,
-	 * worker doing the cleanup, and
-	 * admin reconfiguring us:
-	 * on (re)configure, first set CONFIG_PENDING,
-	 * then wait for a potentially exiting worker,
-	 * start the worker, and schedule one no_op.
-	 * then proceed with configuration.
-	 */
-	if (ns.disk == D_DISKLESS &&
-	    ns.conn == C_STANDALONE &&
-	    ns.role == R_SECONDARY &&
-	    !test_and_set_bit(CONFIG_PENDING, &mdev->flags))
-		set_bit(DEVICE_DYING, &mdev->flags);
-
 	/* if we are going -> D_FAILED or D_DISKLESS, grab one extra reference
 	 * on the ldev here, to be sure the transition -> D_DISKLESS resp.
 	 * drbd_ldev_destroy() won't happen before our corresponding
@@ -768,6 +770,18 @@ __drbd_set_state(struct drbd_conf *mdev, union drbd_state ns,
 
 	mdev->state = ns;
 
+	/* solve the race between becoming unconfigured,
+	 * worker doing the cleanup, and
+	 * admin reconfiguring us:
+	 * on (re)configure, first set CONFIG_PENDING,
+	 * then wait for a potentially exiting worker,
+	 * start the worker, and schedule one no_op.
+	 * then proceed with configuration.
+	 */
+	if(conn_all_vols_unconf(mdev->tconn) &&
+	   !test_and_set_bit(CONFIG_PENDING, &mdev->tconn->flags))
+		set_bit(OBJECT_DYING, &mdev->tconn->flags);
+
 	if (os.disk == D_ATTACHING && ns.disk >= D_NEGOTIATING)
 		drbd_print_uuids(mdev, "attached to UUIDs");
 
@@ -1236,7 +1250,7 @@ static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
 			resume_next_sg(mdev);
 	}
 
-	after_all_state_ch(mdev->tconn, ns);
+	after_all_state_ch(mdev->tconn);
 
 	drbd_md_sync(mdev);
 }
@@ -1248,10 +1262,10 @@ struct after_conn_state_chg_work {
 	enum chg_state_flags flags;
 };
 
-static void after_all_state_ch(struct drbd_tconn *tconn, union drbd_state ns)
+static void after_all_state_ch(struct drbd_tconn *tconn)
 {
-	if (ns.disk == D_DISKLESS && ns.conn == C_STANDALONE && ns.role == R_SECONDARY) {
-		/* if (test_bit(DEVICE_DYING, &mdev->flags)) TODO: DEVICE_DYING functionality */
+	if (conn_all_vols_unconf(tconn) &&
+	    test_bit(OBJECT_DYING, &tconn->flags)) {
 		drbd_thread_stop_nowait(&tconn->worker);
 	}
 }
@@ -1271,7 +1285,7 @@ static int w_after_conn_state_ch(struct drbd_work *w, int unused)
 		drbd_thread_start(&tconn->receiver);
 
 	//conn_err(tconn, STATE_FMT, STATE_ARGS("nms", nms));
-	after_all_state_ch(tconn, nms);
+	after_all_state_ch(tconn);
 
 	return 1;
 }
diff --git a/drivers/block/drbd/drbd_state.h b/drivers/block/drbd/drbd_state.h
index 5fdbdf0..d9536cd 100644
--- a/drivers/block/drbd/drbd_state.h
+++ b/drivers/block/drbd/drbd_state.h
@@ -91,6 +91,7 @@ conn_request_state(struct drbd_tconn *tconn, union drbd_state mask, union drbd_s
 		   enum chg_state_flags flags);
 
 extern void drbd_resume_al(struct drbd_conf *mdev);
+extern int conn_all_vols_unconf(struct drbd_tconn *tconn);
 
 /**
  * drbd_request_state() - Reqest a state change
diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c
index 8ee5c4f..5cb5ffc 100644
--- a/drivers/block/drbd/drbd_worker.c
+++ b/drivers/block/drbd/drbd_worker.c
@@ -1643,29 +1643,13 @@ void drbd_start_resync(struct drbd_conf *mdev, enum drbd_conns side)
 	mutex_unlock(mdev->state_mutex);
 }
 
-static int _worker_dying(int vnr, void *p, void *data)
-{
-	struct drbd_conf *mdev = (struct drbd_conf *)p;
-
-	D_ASSERT(mdev->state.disk == D_DISKLESS && mdev->state.conn == C_STANDALONE);
-	/* _drbd_set_state only uses stop_nowait.
-	 * wait here for the exiting receiver. */
-	drbd_thread_stop(&mdev->tconn->receiver);
-	drbd_mdev_cleanup(mdev);
-
-	clear_bit(DEVICE_DYING, &mdev->flags);
-	clear_bit(CONFIG_PENDING, &mdev->flags);
-	wake_up(&mdev->state_wait);
-
-	return 0;
-}
-
 int drbd_worker(struct drbd_thread *thi)
 {
 	struct drbd_tconn *tconn = thi->tconn;
 	struct drbd_work *w = NULL;
+	struct drbd_conf *mdev;
 	LIST_HEAD(work_list);
-	int intr = 0;
+	int minor, intr = 0;
 
 	while (get_t_state(thi) == RUNNING) {
 		drbd_thread_current_set_cpu(thi);
@@ -1749,7 +1733,16 @@ int drbd_worker(struct drbd_thread *thi)
 	 */
 	spin_unlock_irq(&tconn->data.work.q_lock);
 
-	idr_for_each(&tconn->volumes, _worker_dying, NULL);
+	drbd_thread_stop(&tconn->receiver);
+	idr_for_each_entry(&tconn->volumes, mdev, minor) {
+		D_ASSERT(mdev->state.disk == D_DISKLESS && mdev->state.conn == C_STANDALONE);
+		/* _drbd_set_state only uses stop_nowait.
+		 * wait here for the exiting receiver. */
+		drbd_mdev_cleanup(mdev);
+	}
+	clear_bit(OBJECT_DYING, &tconn->flags);
+	clear_bit(CONFIG_PENDING, &tconn->flags);
+	wake_up(&tconn->ping_wait);
 
 	return 0;
 }
-- 
1.7.4.1

[PATCH 10/16] drbd: Removed the mdev parameter from the ..to_tags() and ...from_tags() functions

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_nl.c |   38 +++++++++++++++++++-------------------
 1 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index 8cdfb46..33159e4 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -50,9 +50,9 @@ static char *drbd_m_holder = "Hands off! this is DRBD's meta data device.";
 
 /* Generate the tag_list to struct functions */
 #define NL_PACKET(name, number, fields) \
-static int name ## _from_tags(struct drbd_conf *mdev, \
+static int name ## _from_tags( \
 	unsigned short *tags, struct name *arg) __attribute__ ((unused)); \
-static int name ## _from_tags(struct drbd_conf *mdev, \
+static int name ## _from_tags( \
 	unsigned short *tags, struct name *arg) \
 { \
 	int tag; \
@@ -64,7 +64,7 @@ static int name ## _from_tags(struct drbd_conf *mdev, \
 		fields \
 		default: \
 			if (tag & T_MANDATORY) { \
-				dev_err(DEV, "Unknown tag: %d\n", tag_number(tag)); \
+				printk(KERN_ERR "drbd: Unknown tag: %d\n", tag_number(tag)); \
 				return 0; \
 			} \
 		} \
@@ -87,7 +87,7 @@ static int name ## _from_tags(struct drbd_conf *mdev, \
 #define NL_STRING(pn, pr, member, len) \
 	case pn: /* D_ASSERT( tag_type(tag) == TT_STRING ); */ \
 		if (dlen > len) { \
-			dev_err(DEV, "arg too long: %s (%u wanted, max len: %u bytes)\n", \
+			printk(KERN_ERR "drbd: arg too long: %s (%u wanted, max len: %u bytes)\n", \
 				#member, dlen, (unsigned int)len); \
 			return 0; \
 		} \
@@ -99,10 +99,10 @@ static int name ## _from_tags(struct drbd_conf *mdev, \
 /* Generate the struct to tag_list functions */
 #define NL_PACKET(name, number, fields) \
 static unsigned short* \
-name ## _to_tags(struct drbd_conf *mdev, \
+name ## _to_tags( \
 	struct name *arg, unsigned short *tags) __attribute__ ((unused)); \
 static unsigned short* \
-name ## _to_tags(struct drbd_conf *mdev, \
+name ## _to_tags( \
 	struct name *arg, unsigned short *tags) \
 { \
 	fields \
@@ -483,7 +483,7 @@ static int drbd_nl_primary(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
 	struct primary primary_args;
 
 	memset(&primary_args, 0, sizeof(struct primary));
-	if (!primary_from_tags(mdev, nlp->tag_list, &primary_args)) {
+	if (!primary_from_tags(nlp->tag_list, &primary_args)) {
 		reply->ret_code = ERR_MANDATORY_TAG;
 		return 0;
 	}
@@ -956,7 +956,7 @@ static int drbd_nl_disk_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp
 	nbc->dc.fencing       = DRBD_FENCING_DEF;
 	nbc->dc.max_bio_bvecs = DRBD_MAX_BIO_BVECS_DEF;
 
-	if (!disk_conf_from_tags(mdev, nlp->tag_list, &nbc->dc)) {
+	if (!disk_conf_from_tags(nlp->tag_list, &nbc->dc)) {
 		retcode = ERR_MANDATORY_TAG;
 		goto fail;
 	}
@@ -1376,7 +1376,7 @@ static int drbd_nl_net_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
 	new_conf->on_congestion    = DRBD_ON_CONGESTION_DEF;
 	new_conf->cong_extents     = DRBD_CONG_EXTENTS_DEF;
 
-	if (!net_conf_from_tags(mdev, nlp->tag_list, new_conf)) {
+	if (!net_conf_from_tags(nlp->tag_list, new_conf)) {
 		retcode = ERR_MANDATORY_TAG;
 		goto fail;
 	}
@@ -1553,7 +1553,7 @@ static int drbd_nl_disconnect(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nl
 	struct disconnect dc;
 
 	memset(&dc, 0, sizeof(struct disconnect));
-	if (!disconnect_from_tags(mdev, nlp->tag_list, &dc)) {
+	if (!disconnect_from_tags(nlp->tag_list, &dc)) {
 		retcode = ERR_MANDATORY_TAG;
 		goto fail;
 	}
@@ -1630,7 +1630,7 @@ static int drbd_nl_resize(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
 	enum dds_flags ddsf;
 
 	memset(&rs, 0, sizeof(struct resize));
-	if (!resize_from_tags(mdev, nlp->tag_list, &rs)) {
+	if (!resize_from_tags(nlp->tag_list, &rs)) {
 		retcode = ERR_MANDATORY_TAG;
 		goto fail;
 	}
@@ -1715,7 +1715,7 @@ static int drbd_nl_syncer_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *n
 	} else
 		memcpy(&sc, &mdev->sync_conf, sizeof(struct syncer_conf));
 
-	if (!syncer_conf_from_tags(mdev, nlp->tag_list, &sc)) {
+	if (!syncer_conf_from_tags(nlp->tag_list, &sc)) {
 		retcode = ERR_MANDATORY_TAG;
 		goto fail;
 	}
@@ -2020,15 +2020,15 @@ static int drbd_nl_get_config(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nl
 	tl = reply->tag_list;
 
 	if (get_ldev(mdev)) {
-		tl = disk_conf_to_tags(mdev, &mdev->ldev->dc, tl);
+		tl = disk_conf_to_tags(&mdev->ldev->dc, tl);
 		put_ldev(mdev);
 	}
 
 	if (get_net_conf(mdev->tconn)) {
-		tl = net_conf_to_tags(mdev, mdev->tconn->net_conf, tl);
+		tl = net_conf_to_tags(mdev->tconn->net_conf, tl);
 		put_net_conf(mdev->tconn);
 	}
-	tl = syncer_conf_to_tags(mdev, &mdev->sync_conf, tl);
+	tl = syncer_conf_to_tags(&mdev->sync_conf, tl);
 
 	put_unaligned(TT_END, tl++); /* Close the tag list */
 
@@ -2043,7 +2043,7 @@ static int drbd_nl_get_state(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp
 	unsigned long rs_left;
 	unsigned int res;
 
-	tl = get_state_to_tags(mdev, (struct get_state *)&s, tl);
+	tl = get_state_to_tags((struct get_state *)&s, tl);
 
 	/* no local ref, no bitmap, no syncer progress. */
 	if (s.conn >= C_SYNC_SOURCE && s.conn <= C_PAUSED_SYNC_T) {
@@ -2105,7 +2105,7 @@ static int drbd_nl_start_ov(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
 	struct start_ov args =
 		{ .start_sector = mdev->ov_start_sector };
 
-	if (!start_ov_from_tags(mdev, nlp->tag_list, &args)) {
+	if (!start_ov_from_tags(nlp->tag_list, &args)) {
 		reply->ret_code = ERR_MANDATORY_TAG;
 		return 0;
 	}
@@ -2131,7 +2131,7 @@ static int drbd_nl_new_c_uuid(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nl
 	struct new_c_uuid args;
 
 	memset(&args, 0, sizeof(struct new_c_uuid));
-	if (!new_c_uuid_from_tags(mdev, nlp->tag_list, &args)) {
+	if (!new_c_uuid_from_tags(nlp->tag_list, &args)) {
 		reply->ret_code = ERR_MANDATORY_TAG;
 		return 0;
 	}
@@ -2365,7 +2365,7 @@ void drbd_bcast_state(struct drbd_conf *mdev, union drbd_state state)
 
 	/* dev_warn(DEV, "drbd_bcast_state() got called\n"); */
 
-	tl = get_state_to_tags(mdev, (struct get_state *)&state, tl);
+	tl = get_state_to_tags((struct get_state *)&state, tl);
 
 	put_unaligned(TT_END, tl++); /* Close the tag list */
 
-- 
1.7.4.1

[PATCH 11/16] drbd: Improved the dec_*() macros

[Philipp Reisner]

Now those can be used with a struct drbd_conf * that has an other
name than 'mdev'.

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_int.h |   51 +++++++++++++++++++++++-----------------
 1 files changed, 29 insertions(+), 22 deletions(-)

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index d53fb0f..004419e 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -1988,17 +1988,19 @@ static inline void inc_ap_pending(struct drbd_conf *mdev)
 	atomic_inc(&mdev->ap_pending_cnt);
 }
 
-#define ERR_IF_CNT_IS_NEGATIVE(which)				\
-	if (atomic_read(&mdev->which) < 0)			\
+#define ERR_IF_CNT_IS_NEGATIVE(which, func, line)			\
+	if (atomic_read(&mdev->which) < 0)				\
 		dev_err(DEV, "in %s:%d: " #which " = %d < 0 !\n",	\
-		    __func__ , __LINE__ ,			\
-		    atomic_read(&mdev->which))
+			func, line,					\
+			atomic_read(&mdev->which))
 
-#define dec_ap_pending(mdev)	do {				\
-	typecheck(struct drbd_conf *, mdev);			\
-	if (atomic_dec_and_test(&mdev->ap_pending_cnt))		\
-		wake_up(&mdev->misc_wait);			\
-	ERR_IF_CNT_IS_NEGATIVE(ap_pending_cnt); } while (0)
+#define dec_ap_pending(mdev) _dec_ap_pending(mdev, __FUNCTION__, __LINE__)
+static inline void _dec_ap_pending(struct drbd_conf *mdev, const char *func, int line)
+{
+	if (atomic_dec_and_test(&mdev->ap_pending_cnt))
+		wake_up(&mdev->misc_wait);
+	ERR_IF_CNT_IS_NEGATIVE(ap_pending_cnt, func, line);
+}
 
 /* counts how many resync-related answers we still expect from the peer
  *		     increase			decrease
@@ -2011,10 +2013,12 @@ static inline void inc_rs_pending(struct drbd_conf *mdev)
 	atomic_inc(&mdev->rs_pending_cnt);
 }
 
-#define dec_rs_pending(mdev)	do {				\
-	typecheck(struct drbd_conf *, mdev);			\
-	atomic_dec(&mdev->rs_pending_cnt);			\
-	ERR_IF_CNT_IS_NEGATIVE(rs_pending_cnt); } while (0)
+#define dec_rs_pending(mdev) _dec_rs_pending(mdev, __FUNCTION__, __LINE__)
+static inline void _dec_rs_pending(struct drbd_conf *mdev, const char *func, int line)
+{
+	atomic_dec(&mdev->rs_pending_cnt);
+	ERR_IF_CNT_IS_NEGATIVE(rs_pending_cnt, func, line);
+}
 
 /* counts how many answers we still need to send to the peer.
  * increased on
@@ -2030,16 +2034,19 @@ static inline void inc_unacked(struct drbd_conf *mdev)
 	atomic_inc(&mdev->unacked_cnt);
 }
 
-#define dec_unacked(mdev)	do {				\
-	typecheck(struct drbd_conf *, mdev);			\
-	atomic_dec(&mdev->unacked_cnt);				\
-	ERR_IF_CNT_IS_NEGATIVE(unacked_cnt); } while (0)
-
-#define sub_unacked(mdev, n)	do {				\
-	typecheck(struct drbd_conf *, mdev);			\
-	atomic_sub(n, &mdev->unacked_cnt);			\
-	ERR_IF_CNT_IS_NEGATIVE(unacked_cnt); } while (0)
+#define dec_unacked(mdev) _dec_unacked(mdev, __FUNCTION__, __LINE__)
+static inline void _dec_unacked(struct drbd_conf *mdev, const char *func, int line)
+{
+	atomic_dec(&mdev->unacked_cnt);
+	ERR_IF_CNT_IS_NEGATIVE(unacked_cnt, func, line);
+}
 
+#define sub_unacked(mdev, n) _sub_unacked(mdev, n, __FUNCTION__, __LINE__)
+static inline void _sub_unacked(struct drbd_conf *mdev, int n, const char *func, int line)
+{
+	atomic_sub(n, &mdev->unacked_cnt);
+	ERR_IF_CNT_IS_NEGATIVE(unacked_cnt, func, line);
+}
 
 static inline void put_net_conf(struct drbd_tconn *tconn)
 {
-- 
1.7.4.1

[PATCH 12/16] drbd: Converted the transfer log from mdev to tconn

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_int.h      |    6 +-
 drivers/block/drbd/drbd_main.c     |  125 +++++++++++++++++++-----------------
 drivers/block/drbd/drbd_nl.c       |    4 +-
 drivers/block/drbd/drbd_receiver.c |    6 +-
 drivers/block/drbd/drbd_req.c      |    2 +-
 drivers/block/drbd/drbd_req.h      |    2 +-
 drivers/block/drbd/drbd_state.c    |    6 +-
 7 files changed, 79 insertions(+), 72 deletions(-)

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index 004419e..fe3c2d2 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -1173,10 +1173,10 @@ extern void drbd_calc_cpu_mask(struct drbd_tconn *tconn);
 #define drbd_calc_cpu_mask(A) ({})
 #endif
 extern void drbd_free_resources(struct drbd_conf *mdev);
-extern void tl_release(struct drbd_conf *mdev, unsigned int barrier_nr,
+extern void tl_release(struct drbd_tconn *, unsigned int barrier_nr,
 		       unsigned int set_size);
-extern void tl_clear(struct drbd_conf *mdev);
-extern void _tl_add_barrier(struct drbd_conf *, struct drbd_tl_epoch *);
+extern void tl_clear(struct drbd_tconn *);
+extern void _tl_add_barrier(struct drbd_tconn *, struct drbd_tl_epoch *);
 extern void drbd_free_sock(struct drbd_tconn *tconn);
 extern int drbd_send(struct drbd_tconn *tconn, struct socket *sock,
 		     void *buf, size_t size, unsigned msg_flags);
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index eb51650..860ed18 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -180,7 +180,7 @@ int _get_ldev_if_state(struct drbd_conf *mdev, enum drbd_disk_state mins)
  * Each &struct drbd_tl_epoch has a circular double linked list of requests
  * attached.
  */
-static int tl_init(struct drbd_conf *mdev)
+static int tl_init(struct drbd_tconn *tconn)
 {
 	struct drbd_tl_epoch *b;
 
@@ -195,21 +195,23 @@ static int tl_init(struct drbd_conf *mdev)
 	b->n_writes = 0;
 	b->w.cb = NULL; /* if this is != NULL, we need to dec_ap_pending in tl_clear */
 
-	mdev->tconn->oldest_tle = b;
-	mdev->tconn->newest_tle = b;
-	INIT_LIST_HEAD(&mdev->tconn->out_of_sequence_requests);
+	tconn->oldest_tle = b;
+	tconn->newest_tle = b;
+	INIT_LIST_HEAD(&tconn->out_of_sequence_requests);
 
 	return 1;
 }
 
-static void tl_cleanup(struct drbd_conf *mdev)
+static void tl_cleanup(struct drbd_tconn *tconn)
 {
-	D_ASSERT(mdev->tconn->oldest_tle == mdev->tconn->newest_tle);
-	D_ASSERT(list_empty(&mdev->tconn->out_of_sequence_requests));
-	kfree(mdev->tconn->oldest_tle);
-	mdev->tconn->oldest_tle = NULL;
-	kfree(mdev->tconn->unused_spare_tle);
-	mdev->tconn->unused_spare_tle = NULL;
+	if (tconn->oldest_tle != tconn->newest_tle)
+		conn_err(tconn, "ASSERT FAILED: oldest_tle == newest_tle\n");
+	if (!list_empty(&tconn->out_of_sequence_requests))
+		conn_err(tconn, "ASSERT FAILED: list_mepty(out_of_sequence_requests)\n");
+	kfree(tconn->oldest_tle);
+	tconn->oldest_tle = NULL;
+	kfree(tconn->unused_spare_tle);
+	tconn->unused_spare_tle = NULL;
 }
 
 /**
@@ -219,7 +221,7 @@ static void tl_cleanup(struct drbd_conf *mdev)
  *
  * The caller must hold the req_lock.
  */
-void _tl_add_barrier(struct drbd_conf *mdev, struct drbd_tl_epoch *new)
+void _tl_add_barrier(struct drbd_tconn *tconn, struct drbd_tl_epoch *new)
 {
 	struct drbd_tl_epoch *newest_before;
 
@@ -229,13 +231,13 @@ void _tl_add_barrier(struct drbd_conf *mdev, struct drbd_tl_epoch *new)
 	new->next = NULL;
 	new->n_writes = 0;
 
-	newest_before = mdev->tconn->newest_tle;
+	newest_before = tconn->newest_tle;
 	/* never send a barrier number == 0, because that is special-cased
 	 * when using TCQ for our write ordering code */
 	new->br_number = (newest_before->br_number+1) ?: 1;
-	if (mdev->tconn->newest_tle != new) {
-		mdev->tconn->newest_tle->next = new;
-		mdev->tconn->newest_tle = new;
+	if (tconn->newest_tle != new) {
+		tconn->newest_tle->next = new;
+		tconn->newest_tle = new;
 	}
 }
 
@@ -249,31 +251,32 @@ void _tl_add_barrier(struct drbd_conf *mdev, struct drbd_tl_epoch *new)
  * &struct drbd_tl_epoch objects this function will cause a termination
  * of the connection.
  */
-void tl_release(struct drbd_conf *mdev, unsigned int barrier_nr,
-		       unsigned int set_size)
+void tl_release(struct drbd_tconn *tconn, unsigned int barrier_nr,
+		unsigned int set_size)
 {
+	struct drbd_conf *mdev;
 	struct drbd_tl_epoch *b, *nob; /* next old barrier */
 	struct list_head *le, *tle;
 	struct drbd_request *r;
 
-	spin_lock_irq(&mdev->tconn->req_lock);
+	spin_lock_irq(&tconn->req_lock);
 
-	b = mdev->tconn->oldest_tle;
+	b = tconn->oldest_tle;
 
 	/* first some paranoia code */
 	if (b == NULL) {
-		dev_err(DEV, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
-			barrier_nr);
+		conn_err(tconn, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
+			 barrier_nr);
 		goto bail;
 	}
 	if (b->br_number != barrier_nr) {
-		dev_err(DEV, "BAD! BarrierAck #%u received, expected #%u!\n",
-			barrier_nr, b->br_number);
+		conn_err(tconn, "BAD! BarrierAck #%u received, expected #%u!\n",
+			 barrier_nr, b->br_number);
 		goto bail;
 	}
 	if (b->n_writes != set_size) {
-		dev_err(DEV, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
-			barrier_nr, set_size, b->n_writes);
+		conn_err(tconn, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
+			 barrier_nr, set_size, b->n_writes);
 		goto bail;
 	}
 
@@ -296,28 +299,29 @@ void tl_release(struct drbd_conf *mdev, unsigned int barrier_nr,
 	   _req_mod(, BARRIER_ACKED) above.
 	   */
 	list_del_init(&b->requests);
+	mdev = b->w.mdev;
 
 	nob = b->next;
 	if (test_and_clear_bit(CREATE_BARRIER, &mdev->flags)) {
-		_tl_add_barrier(mdev, b);
+		_tl_add_barrier(tconn, b);
 		if (nob)
-			mdev->tconn->oldest_tle = nob;
+			tconn->oldest_tle = nob;
 		/* if nob == NULL b was the only barrier, and becomes the new
-		   barrier. Therefore mdev->tconn->oldest_tle points already to b */
+		   barrier. Therefore tconn->oldest_tle points already to b */
 	} else {
 		D_ASSERT(nob != NULL);
-		mdev->tconn->oldest_tle = nob;
+		tconn->oldest_tle = nob;
 		kfree(b);
 	}
 
-	spin_unlock_irq(&mdev->tconn->req_lock);
+	spin_unlock_irq(&tconn->req_lock);
 	dec_ap_pending(mdev);
 
 	return;
 
 bail:
-	spin_unlock_irq(&mdev->tconn->req_lock);
-	drbd_force_state(mdev, NS(conn, C_PROTOCOL_ERROR));
+	spin_unlock_irq(&tconn->req_lock);
+	conn_request_state(tconn, NS(conn, C_PROTOCOL_ERROR), CS_HARD);
 }
 
 
@@ -329,15 +333,15 @@ bail:
  * @what might be one of CONNECTION_LOST_WHILE_PENDING, RESEND, FAIL_FROZEN_DISK_IO,
  * RESTART_FROZEN_DISK_IO.
  */
-void _tl_restart(struct drbd_conf *mdev, enum drbd_req_event what)
+void _tl_restart(struct drbd_tconn *tconn, enum drbd_req_event what)
 {
 	struct drbd_tl_epoch *b, *tmp, **pn;
 	struct list_head *le, *tle, carry_reads;
 	struct drbd_request *req;
 	int rv, n_writes, n_reads;
 
-	b = mdev->tconn->oldest_tle;
-	pn = &mdev->tconn->oldest_tle;
+	b = tconn->oldest_tle;
+	pn = &tconn->oldest_tle;
 	while (b) {
 		n_writes = 0;
 		n_reads = 0;
@@ -356,11 +360,11 @@ void _tl_restart(struct drbd_conf *mdev, enum drbd_req_event what)
 				b->n_writes = n_writes;
 				if (b->w.cb == NULL) {
 					b->w.cb = w_send_barrier;
-					inc_ap_pending(mdev);
-					set_bit(CREATE_BARRIER, &mdev->flags);
+					inc_ap_pending(b->w.mdev);
+					set_bit(CREATE_BARRIER, &b->w.mdev->flags);
 				}
 
-				drbd_queue_work(&mdev->tconn->data.work, &b->w);
+				drbd_queue_work(&tconn->data.work, &b->w);
 			}
 			pn = &b->next;
 		} else {
@@ -374,11 +378,12 @@ void _tl_restart(struct drbd_conf *mdev, enum drbd_req_event what)
 			 * the newest barrier may not have been queued yet,
 			 * in which case w.cb is still NULL. */
 			if (b->w.cb != NULL)
-				dec_ap_pending(mdev);
+				dec_ap_pending(b->w.mdev);
 
-			if (b == mdev->tconn->newest_tle) {
+			if (b == tconn->newest_tle) {
 				/* recycle, but reinit! */
-				D_ASSERT(tmp == NULL);
+				if (tmp != NULL)
+					conn_err(tconn, "ASSERT FAILED tmp == NULL");
 				INIT_LIST_HEAD(&b->requests);
 				list_splice(&carry_reads, &b->requests);
 				INIT_LIST_HEAD(&b->w.list);
@@ -406,20 +411,23 @@ void _tl_restart(struct drbd_conf *mdev, enum drbd_req_event what)
  * by the requests on the transfer gets marked as our of sync. Called from the
  * receiver thread and the worker thread.
  */
-void tl_clear(struct drbd_conf *mdev)
+void tl_clear(struct drbd_tconn *tconn)
 {
+	struct drbd_conf *mdev;
 	struct list_head *le, *tle;
 	struct drbd_request *r;
+	int minor;
 
-	spin_lock_irq(&mdev->tconn->req_lock);
+	spin_lock_irq(&tconn->req_lock);
 
-	_tl_restart(mdev, CONNECTION_LOST_WHILE_PENDING);
+	_tl_restart(tconn, CONNECTION_LOST_WHILE_PENDING);
 
 	/* we expect this list to be empty. */
-	D_ASSERT(list_empty(&mdev->tconn->out_of_sequence_requests));
+	if (!list_empty(&tconn->out_of_sequence_requests))
+		conn_err(tconn, "ASSERT FAILED list_empty(&out_of_sequence_requests)\n");
 
 	/* but just in case, clean it up anyways! */
-	list_for_each_safe(le, tle, &mdev->tconn->out_of_sequence_requests) {
+	list_for_each_safe(le, tle, &tconn->out_of_sequence_requests) {
 		r = list_entry(le, struct drbd_request, tl_requests);
 		/* It would be nice to complete outside of spinlock.
 		 * But this is easier for now. */
@@ -427,16 +435,17 @@ void tl_clear(struct drbd_conf *mdev)
 	}
 
 	/* ensure bit indicating barrier is required is clear */
-	clear_bit(CREATE_BARRIER, &mdev->flags);
+	idr_for_each_entry(&tconn->volumes, mdev, minor)
+		clear_bit(CREATE_BARRIER, &mdev->flags);
 
-	spin_unlock_irq(&mdev->tconn->req_lock);
+	spin_unlock_irq(&tconn->req_lock);
 }
 
-void tl_restart(struct drbd_conf *mdev, enum drbd_req_event what)
+void tl_restart(struct drbd_tconn *tconn, enum drbd_req_event what)
 {
-	spin_lock_irq(&mdev->tconn->req_lock);
-	_tl_restart(mdev, what);
-	spin_unlock_irq(&mdev->tconn->req_lock);
+	spin_lock_irq(&tconn->req_lock);
+	_tl_restart(tconn, what);
+	spin_unlock_irq(&tconn->req_lock);
 }
 
 static int drbd_thread_setup(void *arg)
@@ -2199,6 +2208,9 @@ struct drbd_tconn *drbd_new_tconn(char *name)
 	if (!tconn->name)
 		goto fail;
 
+	if (!tl_init(tconn))
+		goto fail;
+
 	tconn->cstate = C_STANDALONE;
 	mutex_init(&tconn->cstate_mutex);
 	spin_lock_init(&tconn->req_lock);
@@ -2224,6 +2236,7 @@ struct drbd_tconn *drbd_new_tconn(char *name)
 	return tconn;
 
 fail:
+	tl_cleanup(tconn);
 	kfree(tconn->name);
 	kfree(tconn);
 
@@ -2316,9 +2329,6 @@ struct drbd_conf *drbd_new_device(unsigned int minor)
 
 	if (drbd_bm_init(mdev))
 		goto out_no_bitmap;
-	/* no need to lock access, we are still initializing this minor device. */
-	if (!tl_init(mdev))
-		goto out_no_tl;
 	mdev->read_requests = RB_ROOT;
 	mdev->write_requests = RB_ROOT;
 
@@ -2334,8 +2344,6 @@ struct drbd_conf *drbd_new_device(unsigned int minor)
 /* out_whatever_else:
 	kfree(mdev->current_epoch); */
 out_no_epoch:
-	tl_cleanup(mdev);
-out_no_tl:
 	drbd_bm_cleanup(mdev);
 out_no_bitmap:
 	__free_page(mdev->md_io_page);
@@ -2357,7 +2365,6 @@ out_no_tconn:
 void drbd_free_mdev(struct drbd_conf *mdev)
 {
 	kfree(mdev->current_epoch);
-	tl_cleanup(mdev);
 	if (mdev->bitmap) /* should no longer be there. */
 		drbd_bm_cleanup(mdev);
 	__free_page(mdev->md_io_page);
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index 33159e4..b141f89 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -1996,9 +1996,9 @@ static int drbd_nl_resume_io(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp
 	reply->ret_code = drbd_request_state(mdev, NS3(susp, 0, susp_nod, 0, susp_fen, 0));
 	if (reply->ret_code == SS_SUCCESS) {
 		if (mdev->state.conn < C_CONNECTED)
-			tl_clear(mdev);
+			tl_clear(mdev->tconn);
 		if (mdev->state.disk == D_DISKLESS || mdev->state.disk == D_FAILED)
-			tl_restart(mdev, FAIL_FROZEN_DISK_IO);
+			tl_restart(mdev->tconn, FAIL_FROZEN_DISK_IO);
 	}
 	drbd_resume_io(mdev);
 
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index b58dc16..e607503 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -3469,7 +3469,7 @@ static int receive_state(struct drbd_conf *mdev, enum drbd_packet cmd,
 		   for temporal network outages! */
 		spin_unlock_irq(&mdev->tconn->req_lock);
 		dev_err(DEV, "Aborting Connect, can not thaw IO with an only Consistent peer\n");
-		tl_clear(mdev);
+		tl_clear(mdev->tconn);
 		drbd_uuid_new_current(mdev);
 		clear_bit(NEW_CUR_UUID, &mdev->flags);
 		drbd_force_state(mdev, NS2(conn, C_PROTOCOL_ERROR, susp, 0));
@@ -4028,7 +4028,7 @@ static int drbd_disconnected(int vnr, void *p, void *data)
 	mdev->p_uuid = NULL;
 
 	if (!is_susp(mdev->state))
-		tl_clear(mdev);
+		tl_clear(mdev->tconn);
 
 	drbd_md_sync(mdev);
 
@@ -4588,7 +4588,7 @@ static int got_BarrierAck(struct drbd_conf *mdev, enum drbd_packet cmd)
 {
 	struct p_barrier_ack *p = &mdev->tconn->meta.rbuf.barrier_ack;
 
-	tl_release(mdev, p->barrier, be32_to_cpu(p->set_size));
+	tl_release(mdev->tconn, p->barrier, be32_to_cpu(p->set_size));
 
 	if (mdev->state.conn == C_AHEAD &&
 	    atomic_read(&mdev->ap_in_flight) == 0 &&
diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c
index cfa5fba..fa799e3 100644
--- a/drivers/block/drbd/drbd_req.c
+++ b/drivers/block/drbd/drbd_req.c
@@ -885,7 +885,7 @@ allocate_barrier:
 	 * barrier packet, this request is queued within the same spinlock. */
 	if ((remote || send_oos) && mdev->tconn->unused_spare_tle &&
 	    test_and_clear_bit(CREATE_BARRIER, &mdev->flags)) {
-		_tl_add_barrier(mdev, mdev->tconn->unused_spare_tle);
+		_tl_add_barrier(mdev->tconn, mdev->tconn->unused_spare_tle);
 		mdev->tconn->unused_spare_tle = NULL;
 	} else {
 		D_ASSERT(!(remote && rw == WRITE &&
diff --git a/drivers/block/drbd/drbd_req.h b/drivers/block/drbd/drbd_req.h
index 0b3cd41..7cfcf3d 100644
--- a/drivers/block/drbd/drbd_req.h
+++ b/drivers/block/drbd/drbd_req.h
@@ -254,7 +254,7 @@ extern int __req_mod(struct drbd_request *req, enum drbd_req_event what,
 extern void complete_master_bio(struct drbd_conf *mdev,
 		struct bio_and_error *m);
 extern void request_timer_fn(unsigned long data);
-extern void tl_restart(struct drbd_conf *mdev, enum drbd_req_event what);
+extern void tl_restart(struct drbd_tconn *, enum drbd_req_event what);
 
 /* use this if you don't want to deal with calling complete_master_bio()
  * outside the spinlock, e.g. when walking some list on cleanup. */
diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index febefcc..100d48b 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -37,7 +37,7 @@ struct after_state_chg_work {
 	struct completion *done;
 };
 
-extern void _tl_restart(struct drbd_conf *mdev, enum drbd_req_event what);
+extern void _tl_restart(struct drbd_tconn *, enum drbd_req_event what);
 static int w_after_state_ch(struct drbd_work *w, int unused);
 static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
 			   union drbd_state ns, enum chg_state_flags flags);
@@ -1009,7 +1009,7 @@ static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
 	if (ns.susp_fen) {
 		/* case1: The outdate peer handler is successful: */
 		if (os.pdsk > D_OUTDATED  && ns.pdsk <= D_OUTDATED) {
-			tl_clear(mdev);
+			tl_clear(mdev->tconn);
 			if (test_bit(NEW_CUR_UUID, &mdev->flags)) {
 				drbd_uuid_new_current(mdev);
 				clear_bit(NEW_CUR_UUID, &mdev->flags);
@@ -1028,7 +1028,7 @@ static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
 
 	if (what != NOTHING) {
 		spin_lock_irq(&mdev->tconn->req_lock);
-		_tl_restart(mdev, what);
+		_tl_restart(mdev->tconn, what);
 		nsm.i &= mdev->state.i;
 		_drbd_set_state(mdev, nsm, CS_VERBOSE, NULL);
 		spin_unlock_irq(&mdev->tconn->req_lock);
-- 
1.7.4.1

[PATCH 13/16] drbd: Preparing the connector interface to operator on connections

[Philipp Reisner]

Up to now it only operated on minor numbers. Now it can work also
on named connections.

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_int.h  |    1 +
 drivers/block/drbd/drbd_main.c |   15 ++++++
 drivers/block/drbd/drbd_nl.c   |   96 ++++++++++++++++++++++++++--------------
 include/linux/drbd.h           |   19 ++++++--
 4 files changed, 94 insertions(+), 37 deletions(-)

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index fe3c2d2..8ef5b82 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -1479,6 +1479,7 @@ extern void drbd_free_mdev(struct drbd_conf *mdev);
 
 struct drbd_tconn *drbd_new_tconn(char *name);
 extern void drbd_free_tconn(struct drbd_tconn *tconn);
+struct drbd_tconn *conn_by_name(const char *name);
 
 extern int proc_details;
 
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index 860ed18..689f23f 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -2196,6 +2196,21 @@ static void drbd_init_workqueue(struct drbd_work_queue* wq)
 	INIT_LIST_HEAD(&wq->q);
 }
 
+struct drbd_tconn *conn_by_name(const char *name)
+{
+	struct drbd_tconn *tconn;
+
+	write_lock_irq(&global_state_lock);
+	list_for_each_entry(tconn, &drbd_tconns, all_tconn) {
+		if (!strcmp(tconn->name, name))
+			goto found;
+	}
+	tconn = NULL;
+found:
+	write_unlock_irq(&global_state_lock);
+	return tconn;
+}
+
 struct drbd_tconn *drbd_new_tconn(char *name)
 {
 	struct drbd_tconn *tconn;
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index b141f89..27a43d1 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -2184,42 +2184,57 @@ out:
 	return 0;
 }
 
+enum cn_handler_type {
+	CHT_MINOR,
+	CHT_CONN,
+	CHT_CTOR,
+	/* CHT_RES, later */
+};
+
 struct cn_handler_struct {
-	int (*function)(struct drbd_conf *,
-			 struct drbd_nl_cfg_req *,
-			 struct drbd_nl_cfg_reply *);
+	enum cn_handler_type type;
+	union {
+		int (*minor_based)(struct drbd_conf *,
+				   struct drbd_nl_cfg_req *,
+				   struct drbd_nl_cfg_reply *);
+		int (*conn_based)(struct drbd_tconn *,
+				  struct drbd_nl_cfg_req *,
+				  struct drbd_nl_cfg_reply *);
+		int (*constructor)(struct drbd_nl_cfg_req *,
+				   struct drbd_nl_cfg_reply *);
+	};
 	int reply_body_size;
 };
 
 static struct cn_handler_struct cnd_table[] = {
-	[ P_primary ]		= { &drbd_nl_primary,		0 },
-	[ P_secondary ]		= { &drbd_nl_secondary,		0 },
-	[ P_disk_conf ]		= { &drbd_nl_disk_conf,		0 },
-	[ P_detach ]		= { &drbd_nl_detach,		0 },
-	[ P_net_conf ]		= { &drbd_nl_net_conf,		0 },
-	[ P_disconnect ]	= { &drbd_nl_disconnect,	0 },
-	[ P_resize ]		= { &drbd_nl_resize,		0 },
-	[ P_syncer_conf ]	= { &drbd_nl_syncer_conf,	0 },
-	[ P_invalidate ]	= { &drbd_nl_invalidate,	0 },
-	[ P_invalidate_peer ]	= { &drbd_nl_invalidate_peer,	0 },
-	[ P_pause_sync ]	= { &drbd_nl_pause_sync,	0 },
-	[ P_resume_sync ]	= { &drbd_nl_resume_sync,	0 },
-	[ P_suspend_io ]	= { &drbd_nl_suspend_io,	0 },
-	[ P_resume_io ]		= { &drbd_nl_resume_io,		0 },
-	[ P_outdate ]		= { &drbd_nl_outdate,		0 },
-	[ P_get_config ]	= { &drbd_nl_get_config,
+	[ P_primary ]		= { CHT_MINOR, { &drbd_nl_primary },	0 },
+	[ P_secondary ]		= { CHT_MINOR, { &drbd_nl_secondary },	0 },
+	[ P_disk_conf ]		= { CHT_MINOR, { &drbd_nl_disk_conf },	0 },
+	[ P_detach ]		= { CHT_MINOR, { &drbd_nl_detach },	0 },
+	[ P_net_conf ]		= { CHT_MINOR, { &drbd_nl_net_conf },	0 },
+	[ P_disconnect ]	= { CHT_MINOR, { &drbd_nl_disconnect },	0 },
+	[ P_resize ]		= { CHT_MINOR, { &drbd_nl_resize },	0 },
+	[ P_syncer_conf ]	= { CHT_MINOR, { &drbd_nl_syncer_conf },0 },
+	[ P_invalidate ]	= { CHT_MINOR, { &drbd_nl_invalidate },	0 },
+	[ P_invalidate_peer ]	= { CHT_MINOR, { &drbd_nl_invalidate_peer },0 },
+	[ P_pause_sync ]	= { CHT_MINOR, { &drbd_nl_pause_sync },	0 },
+	[ P_resume_sync ]	= { CHT_MINOR, { &drbd_nl_resume_sync },0 },
+	[ P_suspend_io ]	= { CHT_MINOR, { &drbd_nl_suspend_io },	0 },
+	[ P_resume_io ]		= { CHT_MINOR, { &drbd_nl_resume_io },	0 },
+	[ P_outdate ]		= { CHT_MINOR, { &drbd_nl_outdate },	0 },
+	[ P_get_config ]	= { CHT_MINOR, { &drbd_nl_get_config },
 				    sizeof(struct syncer_conf_tag_len_struct) +
 				    sizeof(struct disk_conf_tag_len_struct) +
 				    sizeof(struct net_conf_tag_len_struct) },
-	[ P_get_state ]		= { &drbd_nl_get_state,
+	[ P_get_state ]		= { CHT_MINOR, { &drbd_nl_get_state },
 				    sizeof(struct get_state_tag_len_struct) +
 				    sizeof(struct sync_progress_tag_len_struct)	},
-	[ P_get_uuids ]		= { &drbd_nl_get_uuids,
+	[ P_get_uuids ]		= { CHT_MINOR, { &drbd_nl_get_uuids },
 				    sizeof(struct get_uuids_tag_len_struct) },
-	[ P_get_timeout_flag ]	= { &drbd_nl_get_timeout_flag,
+	[ P_get_timeout_flag ]	= { CHT_MINOR, { &drbd_nl_get_timeout_flag },
 				    sizeof(struct get_timeout_flag_tag_len_struct)},
-	[ P_start_ov ]		= { &drbd_nl_start_ov,		0 },
-	[ P_new_c_uuid ]	= { &drbd_nl_new_c_uuid,	0 },
+	[ P_start_ov ]		= { CHT_MINOR, { &drbd_nl_start_ov },	0 },
+	[ P_new_c_uuid ]	= { CHT_MINOR, { &drbd_nl_new_c_uuid },	0 },
 };
 
 static void drbd_connector_callback(struct cn_msg *req, struct netlink_skb_parms *nsp)
@@ -2229,6 +2244,7 @@ static void drbd_connector_callback(struct cn_msg *req, struct netlink_skb_parms
 	struct cn_msg *cn_reply;
 	struct drbd_nl_cfg_reply *reply;
 	struct drbd_conf *mdev;
+	struct drbd_tconn *tconn;
 	int retcode, rr;
 	int reply_size = sizeof(struct cn_msg)
 		+ sizeof(struct drbd_nl_cfg_reply)
@@ -2244,13 +2260,6 @@ static void drbd_connector_callback(struct cn_msg *req, struct netlink_skb_parms
 		goto fail;
 	}
 
-	mdev = ensure_mdev(nlp->drbd_minor,
-			(nlp->flags & DRBD_NL_CREATE_DEVICE));
-	if (!mdev) {
-		retcode = ERR_MINOR_INVALID;
-		goto fail;
-	}
-
 	if (nlp->packet_type >= P_nl_after_last_packet ||
 	    nlp->packet_type == P_return_code_only) {
 		retcode = ERR_PACKET_NR;
@@ -2260,7 +2269,7 @@ static void drbd_connector_callback(struct cn_msg *req, struct netlink_skb_parms
 	cm = cnd_table + nlp->packet_type;
 
 	/* This may happen if packet number is 0: */
-	if (cm->function == NULL) {
+	if (cm->minor_based == NULL) {
 		retcode = ERR_PACKET_NR;
 		goto fail;
 	}
@@ -2281,7 +2290,28 @@ static void drbd_connector_callback(struct cn_msg *req, struct netlink_skb_parms
 	reply->ret_code = NO_ERROR; /* Might by modified by cm->function. */
 	/* reply->tag_list; might be modified by cm->function. */
 
-	rr = cm->function(mdev, nlp, reply);
+	retcode = ERR_MINOR_INVALID;
+	rr = 0;
+	switch (cm->type) {
+	case CHT_MINOR:
+		mdev = minor_to_mdev(nlp->drbd_minor);
+		if (!mdev)
+			goto fail;
+		rr = cm->minor_based(mdev, nlp, reply);
+		break;
+	case CHT_CONN:
+		tconn = conn_by_name(nlp->obj_name);
+		if (!tconn) {
+			retcode = ERR_CONN_NOT_KNOWN;
+			goto fail;
+		}
+		rr = cm->conn_based(tconn, nlp, reply);
+		break;
+	case CHT_CTOR:
+		rr = cm->constructor(nlp, reply);
+		break;
+	/* case CHT_RES: */
+	}
 
 	cn_reply->id = req->id;
 	cn_reply->seq = req->seq;
diff --git a/include/linux/drbd.h b/include/linux/drbd.h
index 70a688b..7683b4a 100644
--- a/include/linux/drbd.h
+++ b/include/linux/drbd.h
@@ -155,6 +155,7 @@ enum drbd_ret_code {
 	ERR_CONG_NOT_PROTO_A	= 155,
 	ERR_PIC_AFTER_DEP	= 156,
 	ERR_PIC_PEER_DEP	= 157,
+	ERR_CONN_NOT_KNOWN      = 158,
 
 	/* insert new ones above this line */
 	AFTER_LAST_ERR_CODE
@@ -347,8 +348,11 @@ enum drbd_timeout_flag {
 
 /* Start of the new netlink/connector stuff */
 
-#define DRBD_NL_CREATE_DEVICE 0x01
-#define DRBD_NL_SET_DEFAULTS  0x02
+enum drbd_ncr_flags {
+	DRBD_NL_CREATE_DEVICE = 0x01,
+	DRBD_NL_SET_DEFAULTS =  0x02,
+};
+#define DRBD_NL_OBJ_NAME_LEN 32
 
 
 /* For searching a vacant cn_idx value */
@@ -356,8 +360,15 @@ enum drbd_timeout_flag {
 
 struct drbd_nl_cfg_req {
 	int packet_type;
-	unsigned int drbd_minor;
-	int flags;
+	union {
+		struct {
+			unsigned int drbd_minor;
+			enum drbd_ncr_flags flags;
+		};
+		struct {
+			char obj_name[DRBD_NL_OBJ_NAME_LEN];
+		};
+	};
 	unsigned short tag_list[];
 };
 
-- 
1.7.4.1

[PATCH 14/16] drbd: Converted drbd_nl_(net_conf|disconnect)() from mdev to tconn

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_int.h  |    2 +
 drivers/block/drbd/drbd_main.c |    2 +-
 drivers/block/drbd/drbd_nl.c   |  120 ++++++++++++++++++++--------------------
 3 files changed, 63 insertions(+), 61 deletions(-)

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index 8ef5b82..2ef168c 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -170,6 +170,7 @@ drbd_insert_fault(struct drbd_conf *mdev, unsigned int type) {
 
 extern struct drbd_conf **minor_table;
 extern struct ratelimit_state drbd_ratelimit_state;
+extern struct list_head drbd_tconns;
 
 /* on the wire */
 enum drbd_packet {
@@ -1474,6 +1475,7 @@ extern wait_queue_head_t drbd_pp_wait;
 
 extern rwlock_t global_state_lock;
 
+extern int conn_lowest_minor(struct drbd_tconn *tconn);
 extern struct drbd_conf *drbd_new_device(unsigned int minor);
 extern void drbd_free_mdev(struct drbd_conf *mdev);
 
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index 689f23f..004118e 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -615,7 +615,7 @@ char *drbd_task_to_thread_name(struct drbd_tconn *tconn, struct task_struct *tas
 }
 
 #ifdef CONFIG_SMP
-static int conn_lowest_minor(struct drbd_tconn *tconn)
+int conn_lowest_minor(struct drbd_tconn *tconn)
 {
 	int minor = 0;
 	idr_get_next(&tconn->volumes, &minor);
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index 27a43d1..455a51d 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -1326,7 +1326,7 @@ static int drbd_nl_detach(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
 	return 0;
 }
 
-static int drbd_nl_net_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
+static int drbd_nl_net_conf(struct drbd_tconn *tconn, struct drbd_nl_cfg_req *nlp,
 			    struct drbd_nl_cfg_reply *reply)
 {
 	int i;
@@ -1335,16 +1335,17 @@ static int drbd_nl_net_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
 	struct crypto_hash *tfm = NULL;
 	struct crypto_hash *integrity_w_tfm = NULL;
 	struct crypto_hash *integrity_r_tfm = NULL;
-	struct drbd_conf *odev;
+	struct drbd_conf *mdev;
 	char hmac_name[CRYPTO_MAX_ALG_NAME];
 	void *int_dig_out = NULL;
 	void *int_dig_in = NULL;
 	void *int_dig_vv = NULL;
+	struct drbd_tconn *oconn;
 	struct sockaddr *new_my_addr, *new_peer_addr, *taken_addr;
 
-	conn_reconfig_start(mdev->tconn);
+	conn_reconfig_start(tconn);
 
-	if (mdev->state.conn > C_STANDALONE) {
+	if (tconn->cstate > C_STANDALONE) {
 		retcode = ERR_NET_CONFIGURED;
 		goto fail;
 	}
@@ -1387,13 +1388,25 @@ static int drbd_nl_net_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
 		goto fail;
 	}
 
-	if (get_ldev(mdev)) {
-		enum drbd_fencing_p fp = mdev->ldev->dc.fencing;
-		put_ldev(mdev);
-		if (new_conf->wire_protocol == DRBD_PROT_A && fp == FP_STONITH) {
-			retcode = ERR_STONITH_AND_PROT_A;
+	idr_for_each_entry(&tconn->volumes, mdev, i) {
+		if (get_ldev(mdev)) {
+			enum drbd_fencing_p fp = mdev->ldev->dc.fencing;
+			put_ldev(mdev);
+			if (new_conf->wire_protocol == DRBD_PROT_A && fp == FP_STONITH) {
+				retcode = ERR_STONITH_AND_PROT_A;
+				goto fail;
+			}
+		}
+		if (mdev->state.role == R_PRIMARY && new_conf->want_lose) {
+			retcode = ERR_DISCARD;
 			goto fail;
 		}
+		if (!mdev->bitmap) {
+			if(drbd_bm_init(mdev)) {
+				retcode = ERR_NOMEM;
+				goto fail;
+			}
+		}
 	}
 
 	if (new_conf->on_congestion != OC_BLOCK && new_conf->wire_protocol != DRBD_PROT_A) {
@@ -1401,31 +1414,25 @@ static int drbd_nl_net_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
 		goto fail;
 	}
 
-	if (mdev->state.role == R_PRIMARY && new_conf->want_lose) {
-		retcode = ERR_DISCARD;
-		goto fail;
-	}
-
 	retcode = NO_ERROR;
 
 	new_my_addr = (struct sockaddr *)&new_conf->my_addr;
 	new_peer_addr = (struct sockaddr *)&new_conf->peer_addr;
-	for (i = 0; i < minor_count; i++) {
-		odev = minor_to_mdev(i);
-		if (!odev || odev == mdev)
+	list_for_each_entry(oconn, &drbd_tconns, all_tconn) {
+		if (oconn == tconn)
 			continue;
-		if (get_net_conf(odev->tconn)) {
-			taken_addr = (struct sockaddr *)&odev->tconn->net_conf->my_addr;
-			if (new_conf->my_addr_len == odev->tconn->net_conf->my_addr_len &&
+		if (get_net_conf(oconn)) {
+			taken_addr = (struct sockaddr *)&oconn->net_conf->my_addr;
+			if (new_conf->my_addr_len == oconn->net_conf->my_addr_len &&
 			    !memcmp(new_my_addr, taken_addr, new_conf->my_addr_len))
 				retcode = ERR_LOCAL_ADDR;
 
-			taken_addr = (struct sockaddr *)&odev->tconn->net_conf->peer_addr;
-			if (new_conf->peer_addr_len == odev->tconn->net_conf->peer_addr_len &&
+			taken_addr = (struct sockaddr *)&oconn->net_conf->peer_addr;
+			if (new_conf->peer_addr_len == oconn->net_conf->peer_addr_len &&
 			    !memcmp(new_peer_addr, taken_addr, new_conf->peer_addr_len))
 				retcode = ERR_PEER_ADDR;
 
-			put_net_conf(odev->tconn);
+			put_net_conf(oconn);
 			if (retcode != NO_ERROR)
 				goto fail;
 		}
@@ -1470,6 +1477,7 @@ static int drbd_nl_net_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
 
 	((char *)new_conf->shared_secret)[SHARED_SECRET_MAX-1] = 0;
 
+	/* allocation not in the IO path, cqueue thread context */
 	if (integrity_w_tfm) {
 		i = crypto_hash_digestsize(integrity_w_tfm);
 		int_dig_out = kmalloc(i, GFP_KERNEL);
@@ -1489,46 +1497,40 @@ static int drbd_nl_net_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
 		}
 	}
 
-	if (!mdev->bitmap) {
-		if(drbd_bm_init(mdev)) {
-			retcode = ERR_NOMEM;
-			goto fail;
-		}
-	}
-
-	drbd_flush_workqueue(mdev);
-	spin_lock_irq(&mdev->tconn->req_lock);
-	if (mdev->tconn->net_conf != NULL) {
+	conn_flush_workqueue(tconn);
+	spin_lock_irq(&tconn->req_lock);
+	if (tconn->net_conf != NULL) {
 		retcode = ERR_NET_CONFIGURED;
-		spin_unlock_irq(&mdev->tconn->req_lock);
+		spin_unlock_irq(&tconn->req_lock);
 		goto fail;
 	}
-	mdev->tconn->net_conf = new_conf;
+	tconn->net_conf = new_conf;
 
-	mdev->send_cnt = 0;
-	mdev->recv_cnt = 0;
-
-	crypto_free_hash(mdev->tconn->cram_hmac_tfm);
-	mdev->tconn->cram_hmac_tfm = tfm;
+	crypto_free_hash(tconn->cram_hmac_tfm);
+	tconn->cram_hmac_tfm = tfm;
 
-	crypto_free_hash(mdev->tconn->integrity_w_tfm);
-	mdev->tconn->integrity_w_tfm = integrity_w_tfm;
+	crypto_free_hash(tconn->integrity_w_tfm);
+	tconn->integrity_w_tfm = integrity_w_tfm;
 
-	crypto_free_hash(mdev->tconn->integrity_r_tfm);
-	mdev->tconn->integrity_r_tfm = integrity_r_tfm;
+	crypto_free_hash(tconn->integrity_r_tfm);
+	tconn->integrity_r_tfm = integrity_r_tfm;
 
-	kfree(mdev->tconn->int_dig_out);
-	kfree(mdev->tconn->int_dig_in);
-	kfree(mdev->tconn->int_dig_vv);
-	mdev->tconn->int_dig_out=int_dig_out;
-	mdev->tconn->int_dig_in=int_dig_in;
-	mdev->tconn->int_dig_vv=int_dig_vv;
-	retcode = _conn_request_state(mdev->tconn, NS(conn, C_UNCONNECTED), CS_VERBOSE);
-	spin_unlock_irq(&mdev->tconn->req_lock);
+	kfree(tconn->int_dig_out);
+	kfree(tconn->int_dig_in);
+	kfree(tconn->int_dig_vv);
+	tconn->int_dig_out=int_dig_out;
+	tconn->int_dig_in=int_dig_in;
+	tconn->int_dig_vv=int_dig_vv;
+	retcode = _conn_request_state(tconn, NS(conn, C_UNCONNECTED), CS_VERBOSE);
+	spin_unlock_irq(&tconn->req_lock);
 
-	kobject_uevent(&disk_to_dev(mdev->vdisk)->kobj, KOBJ_CHANGE);
+	idr_for_each_entry(&tconn->volumes, mdev, i) {
+		mdev->send_cnt = 0;
+		mdev->recv_cnt = 0;
+		kobject_uevent(&disk_to_dev(mdev->vdisk)->kobj, KOBJ_CHANGE);
+	}
 	reply->ret_code = retcode;
-	conn_reconfig_done(mdev->tconn);
+	conn_reconfig_done(tconn);
 	return 0;
 
 fail:
@@ -1541,14 +1543,13 @@ fail:
 	kfree(new_conf);
 
 	reply->ret_code = retcode;
-	conn_reconfig_done(mdev->tconn);
+	conn_reconfig_done(tconn);
 	return 0;
 }
 
-static int drbd_nl_disconnect(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
+static int drbd_nl_disconnect(struct drbd_tconn *tconn, struct drbd_nl_cfg_req *nlp,
 			      struct drbd_nl_cfg_reply *reply)
 {
-	struct drbd_tconn *tconn = mdev->tconn;
 	int retcode;
 	struct disconnect dc;
 
@@ -1600,7 +1601,6 @@ static int drbd_nl_disconnect(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nl
  done:
 	retcode = NO_ERROR;
  fail:
-	drbd_md_sync(mdev);
 	reply->ret_code = retcode;
 	return 0;
 }
@@ -2211,8 +2211,8 @@ static struct cn_handler_struct cnd_table[] = {
 	[ P_secondary ]		= { CHT_MINOR, { &drbd_nl_secondary },	0 },
 	[ P_disk_conf ]		= { CHT_MINOR, { &drbd_nl_disk_conf },	0 },
 	[ P_detach ]		= { CHT_MINOR, { &drbd_nl_detach },	0 },
-	[ P_net_conf ]		= { CHT_MINOR, { &drbd_nl_net_conf },	0 },
-	[ P_disconnect ]	= { CHT_MINOR, { &drbd_nl_disconnect },	0 },
+	[ P_net_conf ]		= { CHT_CONN,  { .conn_based = &drbd_nl_net_conf },	0 },
+	[ P_disconnect ]	= { CHT_CONN,  { .conn_based = &drbd_nl_disconnect },	0 },
 	[ P_resize ]		= { CHT_MINOR, { &drbd_nl_resize },	0 },
 	[ P_syncer_conf ]	= { CHT_MINOR, { &drbd_nl_syncer_conf },0 },
 	[ P_invalidate ]	= { CHT_MINOR, { &drbd_nl_invalidate },	0 },
-- 
1.7.4.1

[PATCH 15/16] drbd: Implemented new commands to create/delete connections/minors

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_int.h  |    4 +-
 drivers/block/drbd/drbd_main.c |   66 ++++++++++++++-----------
 drivers/block/drbd/drbd_nl.c   |  106 ++++++++++++++++++++++++---------------
 include/linux/drbd.h           |    3 +
 include/linux/drbd_nl.h        |   12 +++++
 5 files changed, 119 insertions(+), 72 deletions(-)

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index 2ef168c..5fb5bd0 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -1258,7 +1258,6 @@ extern int drbd_bmio_clear_n_write(struct drbd_conf *mdev);
 extern void drbd_go_diskless(struct drbd_conf *mdev);
 extern void drbd_ldev_destroy(struct drbd_conf *mdev);
 
-
 /* Meta data layout
    We reserve a 128MB Block (4k aligned)
    * either at the end of the backing device
@@ -1476,8 +1475,9 @@ extern wait_queue_head_t drbd_pp_wait;
 extern rwlock_t global_state_lock;
 
 extern int conn_lowest_minor(struct drbd_tconn *tconn);
-extern struct drbd_conf *drbd_new_device(unsigned int minor);
+enum drbd_ret_code conn_new_minor(struct drbd_tconn *tconn, unsigned int minor, int vnr);
 extern void drbd_free_mdev(struct drbd_conf *mdev);
+extern void drbd_delete_device(unsigned int minor);
 
 struct drbd_tconn *drbd_new_tconn(char *name);
 extern void drbd_free_tconn(struct drbd_tconn *tconn);
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index 004118e..1188fca 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -614,13 +614,16 @@ char *drbd_task_to_thread_name(struct drbd_tconn *tconn, struct task_struct *tas
 	return thi ? thi->name : task->comm;
 }
 
-#ifdef CONFIG_SMP
 int conn_lowest_minor(struct drbd_tconn *tconn)
 {
 	int minor = 0;
-	idr_get_next(&tconn->volumes, &minor);
+
+	if (!idr_get_next(&tconn->volumes, &minor))
+		return -1;
 	return minor;
 }
+
+#ifdef CONFIG_SMP
 /**
  * drbd_calc_cpu_mask() - Generate CPU masks, spread over all CPUs
  * @mdev:	DRBD device.
@@ -2078,15 +2081,16 @@ static void drbd_release_ee_lists(struct drbd_conf *mdev)
 		dev_err(DEV, "%d EEs in net list found!\n", rr);
 }
 
-/* caution. no locking.
- * currently only used from module cleanup code. */
-static void drbd_delete_device(unsigned int minor)
+/* caution. no locking. */
+void drbd_delete_device(unsigned int minor)
 {
 	struct drbd_conf *mdev = minor_to_mdev(minor);
 
 	if (!mdev)
 		return;
 
+	idr_remove(&mdev->tconn->volumes, minor);
+
 	/* paranoia asserts */
 	D_ASSERT(mdev->open_cnt == 0);
 	D_ASSERT(list_empty(&mdev->tconn->data.work.q));
@@ -2101,7 +2105,6 @@ static void drbd_delete_device(unsigned int minor)
 		bdput(mdev->this_bdev);
 
 	drbd_free_resources(mdev);
-	drbd_free_tconn(mdev->tconn);
 
 	drbd_release_ee_lists(mdev);
 
@@ -2223,6 +2226,9 @@ struct drbd_tconn *drbd_new_tconn(char *name)
 	if (!tconn->name)
 		goto fail;
 
+	if (!zalloc_cpumask_var(&tconn->cpu_mask, GFP_KERNEL))
+		goto fail;
+
 	if (!tl_init(tconn))
 		goto fail;
 
@@ -2252,6 +2258,7 @@ struct drbd_tconn *drbd_new_tconn(char *name)
 
 fail:
 	tl_cleanup(tconn);
+	free_cpumask_var(tconn->cpu_mask);
 	kfree(tconn->name);
 	kfree(tconn);
 
@@ -2265,6 +2272,7 @@ void drbd_free_tconn(struct drbd_tconn *tconn)
 	write_unlock_irq(&global_state_lock);
 	idr_destroy(&tconn->volumes);
 
+	free_cpumask_var(tconn->cpu_mask);
 	kfree(tconn->name);
 	kfree(tconn->int_dig_out);
 	kfree(tconn->int_dig_in);
@@ -2272,32 +2280,31 @@ void drbd_free_tconn(struct drbd_tconn *tconn)
 	kfree(tconn);
 }
 
-struct drbd_conf *drbd_new_device(unsigned int minor)
+enum drbd_ret_code conn_new_minor(struct drbd_tconn *tconn, unsigned int minor, int vnr)
 {
 	struct drbd_conf *mdev;
 	struct gendisk *disk;
 	struct request_queue *q;
-	char conn_name[9]; /* drbd1234N */
-	int vnr;
+	int vnr_got = vnr;
+
+	mdev = minor_to_mdev(minor);
+	if (mdev)
+		return ERR_MINOR_EXISTS;
 
 	/* GFP_KERNEL, we are outside of all write-out paths */
 	mdev = kzalloc(sizeof(struct drbd_conf), GFP_KERNEL);
 	if (!mdev)
-		return NULL;
-	sprintf(conn_name, "drbd%d", minor);
-	mdev->tconn = drbd_new_tconn(conn_name);
-	if (!mdev->tconn)
-		goto out_no_tconn;
-	if (!idr_pre_get(&mdev->tconn->volumes, GFP_KERNEL))
-		goto out_no_cpumask;
-	if (idr_get_new(&mdev->tconn->volumes, mdev, &vnr))
-		goto out_no_cpumask;
-	if (vnr != 0) {
-		dev_err(DEV, "vnr = %d\n", vnr);
-		goto out_no_cpumask;
-	}
-	if (!zalloc_cpumask_var(&mdev->tconn->cpu_mask, GFP_KERNEL))
-		goto out_no_cpumask;
+		return ERR_NOMEM;
+
+	mdev->tconn = tconn;
+	if (!idr_pre_get(&tconn->volumes, GFP_KERNEL))
+		goto out_no_idr;
+	if (idr_get_new(&tconn->volumes, mdev, &vnr_got))
+		goto out_no_idr;
+	if (vnr_got != vnr) {
+		dev_err(DEV, "vnr_got (%d) != vnr (%d)\n", vnr_got, vnr);
+		goto out_no_q;
+	}
 
 	mdev->minor = minor;
 
@@ -2354,7 +2361,10 @@ struct drbd_conf *drbd_new_device(unsigned int minor)
 	INIT_LIST_HEAD(&mdev->current_epoch->list);
 	mdev->epochs = 1;
 
-	return mdev;
+	minor_table[minor] = mdev;
+	add_disk(disk);
+
+	return NO_ERROR;
 
 /* out_whatever_else:
 	kfree(mdev->current_epoch); */
@@ -2367,12 +2377,10 @@ out_no_io_page:
 out_no_disk:
 	blk_cleanup_queue(q);
 out_no_q:
-	free_cpumask_var(mdev->tconn->cpu_mask);
-out_no_cpumask:
-	drbd_free_tconn(mdev->tconn);
-out_no_tconn:
+	idr_remove(&tconn->volumes, vnr_got);
+out_no_idr:
 	kfree(mdev);
-	return NULL;
+	return ERR_NOMEM;
 }
 
 /* counterpart of drbd_new_device.
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index 455a51d..f2739fd 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -443,40 +443,6 @@ drbd_set_role(struct drbd_conf *mdev, enum drbd_role new_role, int force)
 	return rv;
 }
 
-static struct drbd_conf *ensure_mdev(int minor, int create)
-{
-	struct drbd_conf *mdev;
-
-	if (minor >= minor_count)
-		return NULL;
-
-	mdev = minor_to_mdev(minor);
-
-	if (!mdev && create) {
-		struct gendisk *disk = NULL;
-		mdev = drbd_new_device(minor);
-
-		spin_lock_irq(&drbd_pp_lock);
-		if (minor_table[minor] == NULL) {
-			minor_table[minor] = mdev;
-			disk = mdev->vdisk;
-			mdev = NULL;
-		} /* else: we lost the race */
-		spin_unlock_irq(&drbd_pp_lock);
-
-		if (disk) /* we won the race above */
-			/* in case we ever add a drbd_delete_device(),
-			 * don't forget the del_gendisk! */
-			add_disk(disk);
-		else /* we lost the race above */
-			drbd_free_mdev(mdev);
-
-		mdev = minor_to_mdev(minor);
-	}
-
-	return mdev;
-}
-
 static int drbd_nl_primary(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
 			   struct drbd_nl_cfg_reply *reply)
 {
@@ -1789,12 +1755,6 @@ static int drbd_nl_syncer_conf(struct drbd_conf *mdev, struct drbd_nl_cfg_req *n
 	if (!expect(sc.al_extents <= DRBD_AL_EXTENTS_MAX))
 		sc.al_extents = DRBD_AL_EXTENTS_MAX;
 
-	/* to avoid spurious errors when configuring minors before configuring
-	 * the minors they depend on: if necessary, first create the minor we
-	 * depend on */
-	if (sc.after >= 0)
-		ensure_mdev(sc.after, 1);
-
 	/* most sanity checks done, try to assign the new sync-after
 	 * dependency.  need to hold the global lock in there,
 	 * to avoid a race in the dependency loop check. */
@@ -2184,13 +2144,73 @@ out:
 	return 0;
 }
 
+static int drbd_nl_new_conn(struct drbd_nl_cfg_req *nlp, struct drbd_nl_cfg_reply *reply)
+{
+	struct new_connection args;
+
+	if (!new_connection_from_tags(nlp->tag_list, &args)) {
+		reply->ret_code = ERR_MANDATORY_TAG;
+		return 0;
+	}
+
+	reply->ret_code = NO_ERROR;
+	if (!drbd_new_tconn(args.name))
+		reply->ret_code = ERR_NOMEM;
+
+	return 0;
+}
+
+static int drbd_nl_new_minor(struct drbd_tconn *tconn,
+		      struct drbd_nl_cfg_req *nlp, struct drbd_nl_cfg_reply *reply)
+{
+	struct new_minor args;
+
+	args.vol_nr = 0;
+	args.minor = 0;
+
+	if (!new_minor_from_tags(nlp->tag_list, &args)) {
+		reply->ret_code = ERR_MANDATORY_TAG;
+		return 0;
+	}
+
+	reply->ret_code = conn_new_minor(tconn, args.minor, args.vol_nr);
+
+	return 0;
+}
+
+static int drbd_nl_del_minor(struct drbd_conf *mdev, struct drbd_nl_cfg_req *nlp,
+			     struct drbd_nl_cfg_reply *reply)
+{
+	if (mdev->state.disk == D_DISKLESS &&
+	    mdev->state.conn == C_STANDALONE &&
+	    mdev->state.role == R_SECONDARY) {
+		drbd_delete_device(mdev_to_minor(mdev));
+		reply->ret_code = NO_ERROR;
+	} else {
+		reply->ret_code = ERR_MINOR_CONFIGURED;
+	}
+	return 0;
+}
+
+static int drbd_nl_del_conn(struct drbd_tconn *tconn,
+			    struct drbd_nl_cfg_req *nlp, struct drbd_nl_cfg_reply *reply)
+{
+	if (conn_lowest_minor(tconn) < 0) {
+		drbd_free_tconn(tconn);
+		reply->ret_code = NO_ERROR;
+	} else {
+		reply->ret_code = ERR_CONN_IN_USE;
+	}
+
+	return 0;
+}
+
 enum cn_handler_type {
 	CHT_MINOR,
 	CHT_CONN,
 	CHT_CTOR,
 	/* CHT_RES, later */
 };
-
 struct cn_handler_struct {
 	enum cn_handler_type type;
 	union {
@@ -2235,6 +2255,10 @@ static struct cn_handler_struct cnd_table[] = {
 				    sizeof(struct get_timeout_flag_tag_len_struct)},
 	[ P_start_ov ]		= { CHT_MINOR, { &drbd_nl_start_ov },	0 },
 	[ P_new_c_uuid ]	= { CHT_MINOR, { &drbd_nl_new_c_uuid },	0 },
+	[ P_new_connection ]	= { CHT_CTOR,  { .constructor = &drbd_nl_new_conn }, 0 },
+	[ P_new_minor ]		= { CHT_CONN,  { .conn_based = &drbd_nl_new_minor }, 0 },
+	[ P_del_minor ]		= { CHT_MINOR, { &drbd_nl_del_minor },	0 },
+	[ P_del_connection ]    = { CHT_CONN,  { .conn_based = &drbd_nl_del_conn }, 0 },
 };
 
 static void drbd_connector_callback(struct cn_msg *req, struct netlink_skb_parms *nsp)
diff --git a/include/linux/drbd.h b/include/linux/drbd.h
index 7683b4a..e192167 100644
--- a/include/linux/drbd.h
+++ b/include/linux/drbd.h
@@ -156,6 +156,9 @@ enum drbd_ret_code {
 	ERR_PIC_AFTER_DEP	= 156,
 	ERR_PIC_PEER_DEP	= 157,
 	ERR_CONN_NOT_KNOWN      = 158,
+	ERR_CONN_IN_USE         = 159,
+	ERR_MINOR_CONFIGURED    = 160,
+	ERR_MINOR_EXISTS	= 161,
 
 	/* insert new ones above this line */
 	AFTER_LAST_ERR_CODE
diff --git a/include/linux/drbd_nl.h b/include/linux/drbd_nl.h
index ab6159e4..1216c7a 100644
--- a/include/linux/drbd_nl.h
+++ b/include/linux/drbd_nl.h
@@ -152,6 +152,18 @@ NL_PACKET(new_c_uuid, 26,
 NL_RESPONSE(return_code_only, 27)
 #endif
 
+NL_PACKET(new_connection, 28, /* CHT_CTOR */
+	NL_STRING(	85,	T_MANDATORY,	name, DRBD_NL_OBJ_NAME_LEN)
+)
+
+NL_PACKET(new_minor, 29, /* CHT_CONN */
+	NL_INTEGER(	86,	T_MANDATORY,	minor)
+	NL_INTEGER(	87,	T_MANDATORY,	vol_nr)
+)
+
+NL_PACKET(del_minor, 30, ) /* CHT_MINOR */
+NL_PACKET(del_connection, 31, ) /* CHT_CONN */
+
 #undef NL_PACKET
 #undef NL_INTEGER
 #undef NL_INT64
-- 
1.7.4.1

[PATCH 16/16] drbd: Replaced the minor_table array by an idr

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_int.h      |    8 +-----
 drivers/block/drbd/drbd_main.c     |   42 ++++++++++++++++++-----------------
 drivers/block/drbd/drbd_proc.c     |   14 +++--------
 drivers/block/drbd/drbd_receiver.c |    2 +-
 drivers/block/drbd/drbd_worker.c   |   10 +------
 5 files changed, 31 insertions(+), 45 deletions(-)

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index 5fb5bd0..e3f542c 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -168,8 +168,8 @@ drbd_insert_fault(struct drbd_conf *mdev, unsigned int type) {
 /* 4th incarnation of the disk layout. */
 #define DRBD_MD_MAGIC (DRBD_MAGIC+4)
 
-extern struct drbd_conf **minor_table;
 extern struct ratelimit_state drbd_ratelimit_state;
+extern struct idr minors;
 extern struct list_head drbd_tconns;
 
 /* on the wire */
@@ -1109,11 +1109,7 @@ struct drbd_conf {
 
 static inline struct drbd_conf *minor_to_mdev(unsigned int minor)
 {
-	struct drbd_conf *mdev;
-
-	mdev = minor < minor_count ? minor_table[minor] : NULL;
-
-	return mdev;
+	return (struct drbd_conf *)idr_find(&minors, minor);
 }
 
 static inline unsigned int mdev_to_minor(struct drbd_conf *mdev)
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index 1188fca..0a2575c 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -74,7 +74,7 @@ MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
 MODULE_DESCRIPTION("drbd - Distributed Replicated Block Device v" REL_VERSION);
 MODULE_VERSION(REL_VERSION);
 MODULE_LICENSE("GPL");
-MODULE_PARM_DESC(minor_count, "Maximum number of drbd devices ("
+MODULE_PARM_DESC(minor_count, "Approximate number of drbd devices ("
 		 __stringify(DRBD_MINOR_COUNT_MIN) "-" __stringify(DRBD_MINOR_COUNT_MAX) ")");
 MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR);
 
@@ -120,7 +120,7 @@ module_param_string(usermode_helper, usermode_helper, sizeof(usermode_helper), 0
 /* in 2.6.x, our device mapping and config info contains our virtual gendisks
  * as member "struct gendisk *vdisk;"
  */
-struct drbd_conf **minor_table;
+struct idr minors;
 struct list_head drbd_tconns;  /* list of struct drbd_tconn */
 
 struct kmem_cache *drbd_request_cache;
@@ -2118,11 +2118,13 @@ void drbd_delete_device(unsigned int minor)
 	 * allocated from drbd_new_device
 	 * and actually free the mdev itself */
 	drbd_free_mdev(mdev);
+	idr_remove(&minors, minor);
 }
 
 static void drbd_cleanup(void)
 {
 	unsigned int i;
+	struct drbd_conf *mdev;
 
 	unregister_reboot_notifier(&drbd_notifier);
 
@@ -2139,17 +2141,13 @@ static void drbd_cleanup(void)
 
 	drbd_nl_cleanup();
 
-	if (minor_table) {
-		i = minor_count;
-		while (i--)
-			drbd_delete_device(i);
-		drbd_destroy_mempools();
-	}
-
-	kfree(minor_table);
-
+	idr_for_each_entry(&minors, mdev, i)
+		drbd_delete_device(i);
+	drbd_destroy_mempools();
 	unregister_blkdev(DRBD_MAJOR, "drbd");
 
+	idr_destroy(&minors);
+
 	printk(KERN_INFO "drbd: module cleanup done.\n");
 }
 
@@ -2286,6 +2284,7 @@ enum drbd_ret_code conn_new_minor(struct drbd_tconn *tconn, unsigned int minor,
 	struct gendisk *disk;
 	struct request_queue *q;
 	int vnr_got = vnr;
+	int minor_got = minor;
 
 	mdev = minor_to_mdev(minor);
 	if (mdev)
@@ -2361,13 +2360,20 @@ enum drbd_ret_code conn_new_minor(struct drbd_tconn *tconn, unsigned int minor,
 	INIT_LIST_HEAD(&mdev->current_epoch->list);
 	mdev->epochs = 1;
 
-	minor_table[minor] = mdev;
+	if (!idr_pre_get(&minors, GFP_KERNEL))
+		goto out_no_minor_idr;
+	if (idr_get_new(&minors, mdev, &minor_got))
+		goto out_no_minor_idr;
+	if (minor_got != minor) {
+		idr_remove(&minors, minor_got);
+		goto out_no_minor_idr;
+	}
 	add_disk(disk);
 
 	return NO_ERROR;
 
-/* out_whatever_else:
-	kfree(mdev->current_epoch); */
+out_no_minor_idr:
+	kfree(mdev->current_epoch);
 out_no_epoch:
 	drbd_bm_cleanup(mdev);
 out_no_bitmap:
@@ -2407,7 +2413,7 @@ int __init drbd_init(void)
 
 	if (minor_count < DRBD_MINOR_COUNT_MIN || minor_count > DRBD_MINOR_COUNT_MAX) {
 		printk(KERN_ERR
-			"drbd: invalid minor_count (%d)\n", minor_count);
+		       "drbd: invalid minor_count (%d)\n", minor_count);
 #ifdef MODULE
 		return -EINVAL;
 #else
@@ -2437,10 +2443,7 @@ int __init drbd_init(void)
 	init_waitqueue_head(&drbd_pp_wait);
 
 	drbd_proc = NULL; /* play safe for drbd_cleanup */
-	minor_table = kzalloc(sizeof(struct drbd_conf *)*minor_count,
-				GFP_KERNEL);
-	if (!minor_table)
-		goto Enomem;
+	idr_init(&minors);
 
 	err = drbd_create_mempools();
 	if (err)
@@ -2461,7 +2464,6 @@ int __init drbd_init(void)
 	printk(KERN_INFO "drbd: %s\n", drbd_buildtag());
 	printk(KERN_INFO "drbd: registered as block device major %d\n",
 		DRBD_MAJOR);
-	printk(KERN_INFO "drbd: minor_table @ 0x%p\n", minor_table);
 
 	return 0; /* Success! */
 
diff --git a/drivers/block/drbd/drbd_proc.c b/drivers/block/drbd/drbd_proc.c
index 4e53cb3..36c9a6c 100644
--- a/drivers/block/drbd/drbd_proc.c
+++ b/drivers/block/drbd/drbd_proc.c
@@ -194,7 +194,7 @@ static void resync_dump_detail(struct seq_file *seq, struct lc_element *e)
 
 static int drbd_seq_show(struct seq_file *seq, void *v)
 {
-	int i, hole = 0;
+	int i, prev_i = -1;
 	const char *sn;
 	struct drbd_conf *mdev;
 
@@ -227,16 +227,10 @@ static int drbd_seq_show(struct seq_file *seq, void *v)
 	 oos .. known out-of-sync kB
 	*/
 
-	for (i = 0; i < minor_count; i++) {
-		mdev = minor_to_mdev(i);
-		if (!mdev) {
-			hole = 1;
-			continue;
-		}
-		if (hole) {
-			hole = 0;
+	idr_for_each_entry(&minors, mdev, i) {
+		if (prev_i != i - 1)
 			seq_printf(seq, "\n");
-		}
+		prev_i = i;
 
 		sn = drbd_conn_str(mdev->state.conn);
 
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index e607503..620f27f 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -284,7 +284,7 @@ static void drbd_pp_free(struct drbd_conf *mdev, struct page *page, int is_net)
 	atomic_t *a = is_net ? &mdev->pp_in_use_by_net : &mdev->pp_in_use;
 	int i;
 
-	if (drbd_pp_vacant > (DRBD_MAX_BIO_SIZE/PAGE_SIZE)*minor_count)
+	if (drbd_pp_vacant > (DRBD_MAX_BIO_SIZE/PAGE_SIZE) * minor_count)
 		i = page_chain_free(page);
 	else {
 		struct page *tmp;
diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c
index 5cb5ffc..e459cb2 100644
--- a/drivers/block/drbd/drbd_worker.c
+++ b/drivers/block/drbd/drbd_worker.c
@@ -1349,10 +1349,7 @@ static int _drbd_pause_after(struct drbd_conf *mdev)
 	struct drbd_conf *odev;
 	int i, rv = 0;
 
-	for (i = 0; i < minor_count; i++) {
-		odev = minor_to_mdev(i);
-		if (!odev)
-			continue;
+	idr_for_each_entry(&minors, odev, i) {
 		if (odev->state.conn == C_STANDALONE && odev->state.disk == D_DISKLESS)
 			continue;
 		if (!_drbd_may_sync_now(odev))
@@ -1374,10 +1371,7 @@ static int _drbd_resume_next(struct drbd_conf *mdev)
 	struct drbd_conf *odev;
 	int i, rv = 0;
 
-	for (i = 0; i < minor_count; i++) {
-		odev = minor_to_mdev(i);
-		if (!odev)
-			continue;
+	idr_for_each_entry(&minors, odev, i) {
 		if (odev->state.conn == C_STANDALONE && odev->state.disk == D_DISKLESS)
 			continue;
 		if (odev->state.aftr_isp) {
-- 
1.7.4.1