[RFC 00/11] drbd: part 9 of adding multiple volume support to drbd

[RFC 00/11] drbd: part 9 of adding multiple volume support to drbd

[Philipp Reisner]

This the first request for review of drbd-8.4. The complete set has 
478 patches. This is the eighth installment containing 11 patches.

The whole set is available here:
  git://git.drbd.org/linux-2.6-drbd.git for-jens

and is jens_for-3.2_drivers...for-jens
and this part is 35b5cb2^...5129245

This patch series contains:
 * Ensure that state transitions that affect all volumes in a connections
   are only logged as state transition of the connections, not for the volumes

[PATCH 01/11] drbd: Introduced drbd_read_state()

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_int.h      |    9 +++++++++
 drivers/block/drbd/drbd_nl.c       |    5 ++---
 drivers/block/drbd/drbd_receiver.c |    6 +++---
 drivers/block/drbd/drbd_state.c    |   15 +++++++--------
 drivers/block/drbd/drbd_state.h    |    6 +++---
 drivers/block/drbd/drbd_worker.c   |    6 +++---
 6 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index 54eb96a..ca07afd 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -1682,6 +1682,15 @@ _drbd_set_state(struct drbd_conf *mdev, union drbd_state ns,
 	return rv;
 }
 
+static inline union drbd_state drbd_read_state(struct drbd_conf *mdev)
+{
+	union drbd_state rv;
+
+	rv = mdev->state;
+
+	return rv;
+}
+
 #define __drbd_chk_io_error(m,f) __drbd_chk_io_error_(m,f, __func__)
 static inline void __drbd_chk_io_error_(struct drbd_conf *mdev, int forcedetach, const char *where)
 {
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index 76b98b8..e43b47a 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -1543,8 +1543,8 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 		drbd_suspend_al(mdev); /* IO is still suspended here... */
 
 	spin_lock_irq(&mdev->tconn->req_lock);
-	os = mdev->state;
-	ns.i = os.i;
+	os = drbd_read_state(mdev);
+	ns = os;
 	/* If MDF_CONSISTENT is not set go into inconsistent state,
 	   otherwise investigate MDF_WasUpToDate...
 	   If MDF_WAS_UP_TO_DATE is not set go into D_OUTDATED disk state,
@@ -1586,7 +1586,6 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 	}
 
 	rv = _drbd_set_state(mdev, ns, CS_VERBOSE, NULL);
-	ns = mdev->state;
 	spin_unlock_irq(&mdev->tconn->req_lock);
 
 	if (rv < SS_SUCCESS)
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index 0b51f29..cd0fe24 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -3476,7 +3476,7 @@ static int receive_state(struct drbd_tconn *tconn, struct packet_info *pi)
 
 	spin_lock_irq(&mdev->tconn->req_lock);
  retry:
-	os = ns = mdev->state;
+	os = ns = drbd_read_state(mdev);
 	spin_unlock_irq(&mdev->tconn->req_lock);
 
 	/* peer says his disk is uptodate, while we think it is inconsistent,
@@ -3562,7 +3562,7 @@ static int receive_state(struct drbd_tconn *tconn, struct packet_info *pi)
 	}
 
 	spin_lock_irq(&mdev->tconn->req_lock);
-	if (mdev->state.i != os.i)
+	if (os.i != drbd_read_state(mdev).i)
 		goto retry;
 	clear_bit(CONSIDER_RESYNC, &mdev->flags);
 	ns.peer = peer_state.role;
@@ -3584,7 +3584,7 @@ static int receive_state(struct drbd_tconn *tconn, struct packet_info *pi)
 		return -EIO;
 	}
 	rv = _drbd_set_state(mdev, ns, cs_flags, NULL);
-	ns = mdev->state;
+	ns = drbd_read_state(mdev);
 	spin_unlock_irq(&mdev->tconn->req_lock);
 
 	if (rv < SS_SUCCESS) {
diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index e2aab0d..2b7ba59 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -145,9 +145,8 @@ drbd_change_state(struct drbd_conf *mdev, enum chg_state_flags f,
 	enum drbd_state_rv rv;
 
 	spin_lock_irqsave(&mdev->tconn->req_lock, flags);
-	ns = apply_mask_val(mdev->state, mask, val);
+	ns = apply_mask_val(drbd_read_state(mdev), mask, val);
 	rv = _drbd_set_state(mdev, ns, f, NULL);
-	ns = mdev->state;
 	spin_unlock_irqrestore(&mdev->tconn->req_lock, flags);
 
 	return rv;
@@ -180,7 +179,7 @@ _req_st_cond(struct drbd_conf *mdev, union drbd_state mask,
 		return SS_CW_FAILED_BY_PEER;
 
 	spin_lock_irqsave(&mdev->tconn->req_lock, flags);
-	os = mdev->state;
+	os = drbd_read_state(mdev);
 	ns = sanitize_state(mdev, apply_mask_val(os, mask, val), NULL);
 	rv = is_valid_transition(os, ns);
 	if (rv == SS_SUCCESS)
@@ -226,7 +225,7 @@ drbd_req_state(struct drbd_conf *mdev, union drbd_state mask,
 		mutex_lock(mdev->state_mutex);
 
 	spin_lock_irqsave(&mdev->tconn->req_lock, flags);
-	os = mdev->state;
+	os = drbd_read_state(mdev);
 	ns = sanitize_state(mdev, apply_mask_val(os, mask, val), NULL);
 	rv = is_valid_transition(os, ns);
 	if (rv < SS_SUCCESS) {
@@ -262,7 +261,7 @@ drbd_req_state(struct drbd_conf *mdev, union drbd_state mask,
 			goto abort;
 		}
 		spin_lock_irqsave(&mdev->tconn->req_lock, flags);
-		ns = apply_mask_val(mdev->state, mask, val);
+		ns = apply_mask_val(drbd_read_state(mdev), mask, val);
 		rv = _drbd_set_state(mdev, ns, f, &done);
 	} else {
 		rv = _drbd_set_state(mdev, ns, f, &done);
@@ -794,7 +793,7 @@ __drbd_set_state(struct drbd_conf *mdev, union drbd_state ns,
 	const char *warn_sync_abort = NULL;
 	struct after_state_chg_work *ascw;
 
-	os = mdev->state;
+	os = drbd_read_state(mdev);
 
 	ns = sanitize_state(mdev, ns, &warn_sync_abort);
 	if (ns.i == os.i)
@@ -1412,7 +1411,7 @@ conn_is_valid_transition(struct drbd_tconn *tconn, union drbd_state mask, union
 	int vnr;
 
 	idr_for_each_entry(&tconn->volumes, mdev, vnr) {
-		os = mdev->state;
+		os = drbd_read_state(mdev);
 		ns = sanitize_state(mdev, apply_mask_val(os, mask, val), NULL);
 
 		if (flags & CS_IGN_OUTD_FAIL && ns.disk == D_OUTDATED && os.disk < D_OUTDATED)
@@ -1456,7 +1455,7 @@ conn_set_state(struct drbd_tconn *tconn, union drbd_state mask, union drbd_state
 		tconn->cstate = val.conn;
 
 	idr_for_each_entry(&tconn->volumes, mdev, vnr) {
-		os = mdev->state;
+		os = drbd_read_state(mdev);
 		ns = apply_mask_val(os, mask, val);
 		ns = sanitize_state(mdev, ns, NULL);
 
diff --git a/drivers/block/drbd/drbd_state.h b/drivers/block/drbd/drbd_state.h
index c0331f1..fdcfab9 100644
--- a/drivers/block/drbd/drbd_state.h
+++ b/drivers/block/drbd/drbd_state.h
@@ -48,12 +48,12 @@ struct drbd_tconn;
 	  val.T2 = (S2); val.T3 = (S3); val; })
 
 #define _NS(D, T, S) \
-	D, ({ union drbd_state __ns; __ns.i = D->state.i; __ns.T = (S); __ns; })
+	D, ({ union drbd_state __ns; __ns = drbd_read_state(D); __ns.T = (S); __ns; })
 #define _NS2(D, T1, S1, T2, S2) \
-	D, ({ union drbd_state __ns; __ns.i = D->state.i; __ns.T1 = (S1); \
+	D, ({ union drbd_state __ns; __ns = drbd_read_state(D); __ns.T1 = (S1); \
 	__ns.T2 = (S2); __ns; })
 #define _NS3(D, T1, S1, T2, S2, T3, S3) \
-	D, ({ union drbd_state __ns; __ns.i = D->state.i; __ns.T1 = (S1); \
+	D, ({ union drbd_state __ns; __ns = drbd_read_state(D); __ns.T1 = (S1); \
 	__ns.T2 = (S2); __ns.T3 = (S3); __ns; })
 
 enum chg_state_flags {
diff --git a/drivers/block/drbd/drbd_worker.c b/drivers/block/drbd/drbd_worker.c
index 640d04a..8059ea3 100644
--- a/drivers/block/drbd/drbd_worker.c
+++ b/drivers/block/drbd/drbd_worker.c
@@ -781,7 +781,7 @@ int drbd_resync_finished(struct drbd_conf *mdev)
 	ping_peer(mdev);
 
 	spin_lock_irq(&mdev->tconn->req_lock);
-	os = mdev->state;
+	os = drbd_read_state(mdev);
 
 	verify_done = (os.conn == C_VERIFY_S || os.conn == C_VERIFY_T);
 
@@ -1548,7 +1548,7 @@ void drbd_start_resync(struct drbd_conf *mdev, enum drbd_conns side)
 	}
 
 	write_lock_irq(&global_state_lock);
-	ns = mdev->state;
+	ns = drbd_read_state(mdev);
 
 	ns.aftr_isp = !_drbd_may_sync_now(mdev);
 
@@ -1560,7 +1560,7 @@ void drbd_start_resync(struct drbd_conf *mdev, enum drbd_conns side)
 		ns.pdsk = D_INCONSISTENT;
 
 	r = __drbd_set_state(mdev, ns, CS_VERBOSE, NULL);
-	ns = mdev->state;
+	ns = drbd_read_state(mdev);
 
 	if (ns.conn < C_CONNECTED)
 		r = SS_UNKNOWN_ERROR;
-- 
1.7.4.1

[PATCH 02/11] drbd: Renamed id_susp(union drbd_state s) to drbd_suspended(struct drbd_conf *)

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_int.h      |    6 +++---
 drivers/block/drbd/drbd_nl.c       |    4 ++--
 drivers/block/drbd/drbd_proc.c     |    2 +-
 drivers/block/drbd/drbd_receiver.c |    4 ++--
 drivers/block/drbd/drbd_req.c      |    6 +++---
 drivers/block/drbd/drbd_state.c    |    7 ++++++-
 6 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index ca07afd..4d17f9b 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -2217,16 +2217,16 @@ static inline int drbd_state_is_stable(struct drbd_conf *mdev)
 	return 1;
 }
 
-static inline int is_susp(union drbd_state s)
+static inline int drbd_suspended(struct drbd_conf *mdev)
 {
-	return s.susp || s.susp_nod || s.susp_fen;
+	return mdev->state.susp || mdev->state.susp_nod || mdev->state.susp_fen;
 }
 
 static inline bool may_inc_ap_bio(struct drbd_conf *mdev)
 {
 	int mxb = drbd_get_max_buffers(mdev);
 
-	if (is_susp(mdev->state))
+	if (drbd_suspended(mdev))
 		return false;
 	if (test_bit(SUSPEND_IO, &mdev->flags))
 		return false;
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index e43b47a..57f320e 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -757,7 +757,7 @@ char *ppsize(char *buf, unsigned long long size)
 void drbd_suspend_io(struct drbd_conf *mdev)
 {
 	set_bit(SUSPEND_IO, &mdev->flags);
-	if (is_susp(mdev->state))
+	if (drbd_suspended(mdev))
 		return;
 	wait_event(mdev->misc_wait, !atomic_read(&mdev->ap_bio_cnt));
 }
@@ -1395,7 +1395,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 
 	drbd_suspend_io(mdev);
 	/* also wait for the last barrier ack. */
-	wait_event(mdev->misc_wait, !atomic_read(&mdev->ap_pending_cnt) || is_susp(mdev->state));
+	wait_event(mdev->misc_wait, !atomic_read(&mdev->ap_pending_cnt) || drbd_suspended(mdev));
 	/* and for any other previously queued work */
 	drbd_flush_workqueue(mdev);
 
diff --git a/drivers/block/drbd/drbd_proc.c b/drivers/block/drbd/drbd_proc.c
index 36c9a6c..a4dbdbc 100644
--- a/drivers/block/drbd/drbd_proc.c
+++ b/drivers/block/drbd/drbd_proc.c
@@ -250,7 +250,7 @@ static int drbd_seq_show(struct seq_file *seq, void *v)
 			   drbd_disk_str(mdev->state.pdsk),
 			   (mdev->tconn->net_conf == NULL ? ' ' :
 			    (mdev->tconn->net_conf->wire_protocol - DRBD_PROT_A+'A')),
-			   is_susp(mdev->state) ? 's' : 'r',
+			   drbd_suspended(mdev) ? 's' : 'r',
 			   mdev->state.aftr_isp ? 'a' : '-',
 			   mdev->state.peer_isp ? 'p' : '-',
 			   mdev->state.user_isp ? 'u' : '-',
diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index cd0fe24..d6afb3a 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -3571,7 +3571,7 @@ static int receive_state(struct drbd_tconn *tconn, struct packet_info *pi)
 	if ((ns.conn == C_CONNECTED || ns.conn == C_WF_BITMAP_S) && ns.disk == D_NEGOTIATING)
 		ns.disk = mdev->new_state_tmp.disk;
 	cs_flags = CS_VERBOSE + (os.conn < C_CONNECTED && ns.conn >= C_CONNECTED ? 0 : CS_HARD);
-	if (ns.pdsk == D_CONSISTENT && is_susp(ns) && ns.conn == C_CONNECTED && os.conn < C_CONNECTED &&
+	if (ns.pdsk == D_CONSISTENT && drbd_suspended(mdev) && ns.conn == C_CONNECTED && os.conn < C_CONNECTED &&
 	    test_bit(NEW_CUR_UUID, &mdev->flags)) {
 		/* Do not allow tl_restart(RESEND) for a rebooted peer. We can only allow this
 		   for temporal network outages! */
@@ -4126,7 +4126,7 @@ static int drbd_disconnected(int vnr, void *p, void *data)
 	kfree(mdev->p_uuid);
 	mdev->p_uuid = NULL;
 
-	if (!is_susp(mdev->state))
+	if (!drbd_suspended(mdev))
 		tl_clear(mdev->tconn);
 
 	drbd_md_sync(mdev);
diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c
index 6e0e3bb..e296d7e 100644
--- a/drivers/block/drbd/drbd_req.c
+++ b/drivers/block/drbd/drbd_req.c
@@ -303,7 +303,7 @@ static void _req_may_be_done_not_susp(struct drbd_request *req, struct bio_and_e
 {
 	struct drbd_conf *mdev = req->w.mdev;
 
-	if (!is_susp(mdev->state))
+	if (!drbd_suspended(mdev))
 		_req_may_be_done(req, m);
 }
 
@@ -789,7 +789,7 @@ int __drbd_make_request(struct drbd_conf *mdev, struct bio *bio, unsigned long s
 	send_oos = rw == WRITE && drbd_should_send_out_of_sync(mdev->state);
 	D_ASSERT(!(remote && send_oos));
 
-	if (!(local || remote) && !is_susp(mdev->state)) {
+	if (!(local || remote) && !drbd_suspended(mdev)) {
 		if (__ratelimit(&drbd_ratelimit_state))
 			dev_err(DEV, "IO ERROR: neither local nor remote disk\n");
 		err = -EIO;
@@ -830,7 +830,7 @@ allocate_barrier:
 		}
 	}
 
-	if (is_susp(mdev->state)) {
+	if (drbd_suspended(mdev)) {
 		/* If we got suspended, use the retry mechanism of
 		   generic_make_request() to restart processing of this
 		   bio. In the next call to drbd_make_request
diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index 2b7ba59..c9d9640 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -48,6 +48,11 @@ static enum drbd_state_rv is_valid_transition(union drbd_state os, union drbd_st
 static union drbd_state sanitize_state(struct drbd_conf *mdev, union drbd_state ns,
 				       const char **warn_sync_abort);
 
+static inline bool is_susp(union drbd_state s)
+{
+        return s.susp || s.susp_nod || s.susp_fen;
+}
+
 bool conn_all_vols_unconf(struct drbd_tconn *tconn)
 {
 	struct drbd_conf *mdev;
@@ -1143,7 +1148,7 @@ static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
 		if (get_ldev(mdev)) {
 			if ((ns.role == R_PRIMARY || ns.peer == R_PRIMARY) &&
 			    mdev->ldev->md.uuid[UI_BITMAP] == 0 && ns.disk >= D_UP_TO_DATE) {
-				if (is_susp(mdev->state)) {
+				if (drbd_suspended(mdev)) {
 					set_bit(NEW_CUR_UUID, &mdev->flags);
 				} else {
 					drbd_uuid_new_current(mdev);
-- 
1.7.4.1

[PATCH 03/11] drbd: Moved susp, susp_nod and susp_fen to the connection object

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_int.h   |   10 +++++++++-
 drivers/block/drbd/drbd_state.c |    3 +++
 2 files changed, 12 insertions(+), 1 deletions(-)

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index 4d17f9b..e33a770 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -862,6 +862,9 @@ struct drbd_tconn {			/* is a resource from the config file */
 	struct list_head all_tconn;	/* linked on global drbd_tconns */
 	struct idr volumes;		/* <tconn, vnr> to mdev mapping */
 	enum drbd_conns cstate;		/* Only C_STANDALONE to C_WF_REPORT_PARAMS */
+	unsigned susp:1;		/* IO suspended by user */
+	unsigned susp_nod:1;		/* IO suspended because no data */
+	unsigned susp_fen:1;		/* IO suspended because fence peer handler runs */
 	struct mutex cstate_mutex;	/* Protects graceful disconnects */
 
 	unsigned long flags;
@@ -1687,6 +1690,9 @@ static inline union drbd_state drbd_read_state(struct drbd_conf *mdev)
 	union drbd_state rv;
 
 	rv = mdev->state;
+	rv.susp = mdev->tconn->susp;
+	rv.susp_nod = mdev->tconn->susp_nod;
+	rv.susp_fen = mdev->tconn->susp_fen;
 
 	return rv;
 }
@@ -2219,7 +2225,9 @@ static inline int drbd_state_is_stable(struct drbd_conf *mdev)
 
 static inline int drbd_suspended(struct drbd_conf *mdev)
 {
-	return mdev->state.susp || mdev->state.susp_nod || mdev->state.susp_fen;
+	struct drbd_tconn *tconn = mdev->tconn;
+
+	return tconn->susp || tconn->susp_fen || tconn->susp_nod;
 }
 
 static inline bool may_inc_ap_bio(struct drbd_conf *mdev)
diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index c9d9640..9354006 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -843,6 +843,9 @@ __drbd_set_state(struct drbd_conf *mdev, union drbd_state ns,
 		atomic_inc(&mdev->local_cnt);
 
 	mdev->state = ns;
+	mdev->tconn->susp = ns.susp;
+	mdev->tconn->susp_nod = ns.susp_nod;
+	mdev->tconn->susp_fen = ns.susp_fen;
 
 	/* solve the race between becoming unconfigured,
 	 * worker doing the cleanup, and
-- 
1.7.4.1

[PATCH 04/11] drbd: Introduced a new type union drbd_dev_state

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_int.h   |    6 +++---
 drivers/block/drbd/drbd_main.c  |    5 +----
 drivers/block/drbd/drbd_nl.c    |    4 ++--
 drivers/block/drbd/drbd_req.h   |    4 ++--
 drivers/block/drbd/drbd_state.c |    6 +++---
 drivers/block/drbd/drbd_state.h |   34 ++++++++++++++++++++++++++++++++++
 6 files changed, 45 insertions(+), 14 deletions(-)

diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index e33a770..912ee5e 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -935,7 +935,7 @@ struct drbd_conf {
 	/* Used after attach while negotiating new disk state. */
 	union drbd_state new_state_tmp;
 
-	union drbd_state state;
+	union drbd_dev_state state;
 	wait_queue_head_t misc_wait;
 	wait_queue_head_t state_wait;  /* upon each state change. */
 	unsigned int send_cnt;
@@ -1689,7 +1689,7 @@ static inline union drbd_state drbd_read_state(struct drbd_conf *mdev)
 {
 	union drbd_state rv;
 
-	rv = mdev->state;
+	rv.i = mdev->state.i;
 	rv.susp = mdev->tconn->susp;
 	rv.susp_nod = mdev->tconn->susp_nod;
 	rv.susp_fen = mdev->tconn->susp_fen;
@@ -2155,7 +2155,7 @@ static inline int drbd_get_max_buffers(struct drbd_conf *mdev)
 
 static inline int drbd_state_is_stable(struct drbd_conf *mdev)
 {
-	union drbd_state s = mdev->state;
+	union drbd_dev_state s = mdev->state;
 
 	/* DO NOT add a default clause, we want the compiler to warn us
 	 * for any newly introduced state we may have forgotten to add here */
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index a773d73..a6459f7 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -1823,15 +1823,12 @@ static void drbd_set_defaults(struct drbd_conf *mdev)
 {
 	/* Beware! The actual layout differs
 	 * between big endian and little endian */
-	mdev->state = (union drbd_state) {
+	mdev->state = (union drbd_dev_state) {
 		{ .role = R_SECONDARY,
 		  .peer = R_UNKNOWN,
 		  .conn = C_STANDALONE,
 		  .disk = D_DISKLESS,
 		  .pdsk = D_UNKNOWN,
-		  .susp = 0,
-		  .susp_nod = 0,
-		  .susp_fen = 0
 		} };
 }
 
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index 57f320e..02d4224 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -1475,7 +1475,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
 		clear_bit(CRASHED_PRIMARY, &mdev->flags);
 
 	if (drbd_md_test_flag(mdev->ldev, MDF_PRIMARY_IND) &&
-	    !(mdev->state.role == R_PRIMARY && mdev->state.susp_nod)) {
+	    !(mdev->state.role == R_PRIMARY && mdev->tconn->susp_nod)) {
 		set_bit(CRASHED_PRIMARY, &mdev->flags);
 		cp_discovered = 1;
 	}
@@ -2402,7 +2402,7 @@ out:
 
 int drbd_adm_resume_sync(struct sk_buff *skb, struct genl_info *info)
 {
-	union drbd_state s;
+	union drbd_dev_state s;
 	enum drbd_ret_code retcode;
 
 	retcode = drbd_adm_prepare(skb, info, DRBD_ADM_NEED_MINOR);
diff --git a/drivers/block/drbd/drbd_req.h b/drivers/block/drbd/drbd_req.h
index 4e1f129..b9f474a 100644
--- a/drivers/block/drbd/drbd_req.h
+++ b/drivers/block/drbd/drbd_req.h
@@ -294,7 +294,7 @@ static inline int req_mod(struct drbd_request *req,
 	return rv;
 }
 
-static inline bool drbd_should_do_remote(union drbd_state s)
+static inline bool drbd_should_do_remote(union drbd_dev_state s)
 {
 	return s.pdsk == D_UP_TO_DATE ||
 		(s.pdsk >= D_INCONSISTENT &&
@@ -304,7 +304,7 @@ static inline bool drbd_should_do_remote(union drbd_state s)
 	   That is equivalent since before 96 IO was frozen in the C_WF_BITMAP*
 	   states. */
 }
-static inline bool drbd_should_send_out_of_sync(union drbd_state s)
+static inline bool drbd_should_send_out_of_sync(union drbd_dev_state s)
 {
 	return s.conn == C_AHEAD || s.conn == C_WF_BITMAP_S;
 	/* pdsk = D_INCONSISTENT as a consequence. Protocol 96 check not necessary
diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index 9354006..64284b8 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -842,7 +842,7 @@ __drbd_set_state(struct drbd_conf *mdev, union drbd_state ns,
 	    (os.disk != D_DISKLESS && ns.disk == D_DISKLESS))
 		atomic_inc(&mdev->local_cnt);
 
-	mdev->state = ns;
+	mdev->state.i = ns.i;
 	mdev->tconn->susp = ns.susp;
 	mdev->tconn->susp_nod = ns.susp_nod;
 	mdev->tconn->susp_fen = ns.susp_fen;
@@ -1375,7 +1375,7 @@ static int w_after_conn_state_ch(struct drbd_work *w, int unused)
 void conn_old_common_state(struct drbd_tconn *tconn, union drbd_state *pcs, enum chg_state_flags *pf)
 {
 	enum chg_state_flags flags = ~0;
-	union drbd_state os, cs = {}; /* old_state, common_state */
+	union drbd_dev_state os, cs = {}; /* old_state, common_state */
 	struct drbd_conf *mdev;
 	int vnr, first_vol = 1;
 
@@ -1406,7 +1406,7 @@ void conn_old_common_state(struct drbd_tconn *tconn, union drbd_state *pcs, enum
 
 	*pf |= CS_DC_MASK;
 	*pf &= flags;
-	*pcs = cs;
+	(*pcs).i = cs.i;
 }
 
 static enum drbd_state_rv
diff --git a/drivers/block/drbd/drbd_state.h b/drivers/block/drbd/drbd_state.h
index fdcfab9..0f8441d 100644
--- a/drivers/block/drbd/drbd_state.h
+++ b/drivers/block/drbd/drbd_state.h
@@ -72,6 +72,40 @@ enum chg_state_flags {
 	CS_IGN_OUTD_FAIL = 1 << 10,
 };
 
+/* drbd_dev_state and drbd_state are different types. This is to stress the
+   small difference. There is no suspended flag (.susp), and no suspended
+   while fence handler runs flas (susp_fen). */
+union drbd_dev_state {
+	struct {
+#if defined(__LITTLE_ENDIAN_BITFIELD)
+		unsigned role:2 ;   /* 3/4	 primary/secondary/unknown */
+		unsigned peer:2 ;   /* 3/4	 primary/secondary/unknown */
+		unsigned conn:5 ;   /* 17/32	 cstates */
+		unsigned disk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
+		unsigned pdsk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
+		unsigned _unused:1 ;
+		unsigned aftr_isp:1 ; /* isp .. imposed sync pause */
+		unsigned peer_isp:1 ;
+		unsigned user_isp:1 ;
+		unsigned _pad:11;   /* 0	 unused */
+#elif defined(__BIG_ENDIAN_BITFIELD)
+		unsigned _pad:11;
+		unsigned user_isp:1 ;
+		unsigned peer_isp:1 ;
+		unsigned aftr_isp:1 ; /* isp .. imposed sync pause */
+		unsigned _unused:1 ;
+		unsigned pdsk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
+		unsigned disk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
+		unsigned conn:5 ;   /* 17/32	 cstates */
+		unsigned peer:2 ;   /* 3/4	 primary/secondary/unknown */
+		unsigned role:2 ;   /* 3/4	 primary/secondary/unknown */
+#else
+# error "this endianess is not supported"
+#endif
+	};
+	unsigned int i;
+};
+
 extern enum drbd_state_rv drbd_change_state(struct drbd_conf *mdev,
 					    enum chg_state_flags f,
 					    union drbd_state mask,
-- 
1.7.4.1

[PATCH 05/11] drbd: Renamed nms to ns_max

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_state.c |   16 ++++++++--------
 1 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index 64284b8..bd50644 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -1340,7 +1340,7 @@ static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
 struct after_conn_state_chg_work {
 	struct drbd_work w;
 	enum drbd_conns oc;
-	union drbd_state nms; /* new, max state, over all mdevs */
+	union drbd_state ns_max; /* new, max state, over all mdevs */
 	enum chg_state_flags flags;
 };
 
@@ -1358,12 +1358,12 @@ static int w_after_conn_state_ch(struct drbd_work *w, int unused)
 		container_of(w, struct after_conn_state_chg_work, w);
 	struct drbd_tconn *tconn = w->tconn;
 	enum drbd_conns oc = acscw->oc;
-	union drbd_state nms = acscw->nms;
+	union drbd_state ns_max = acscw->ns_max;
 
 	kfree(acscw);
 
 	/* Upon network configuration, we need to start the receiver */
-	if (oc == C_STANDALONE && nms.conn == C_UNCONNECTED)
+	if (oc == C_STANDALONE && ns_max.conn == C_UNCONNECTED)
 		drbd_thread_start(&tconn->receiver);
 
 	//conn_err(tconn, STATE_FMT, STATE_ARGS("nms", nms));
@@ -1541,7 +1541,7 @@ _conn_request_state(struct drbd_tconn *tconn, union drbd_state mask, union drbd_
 	enum drbd_state_rv rv = SS_SUCCESS;
 	struct after_conn_state_chg_work *acscw;
 	enum drbd_conns oc = tconn->cstate;
-	union drbd_state ms, os;
+	union drbd_state ns_max, os;
 
 	rv = is_valid_conn_transition(oc, val.conn);
 	if (rv < SS_SUCCESS)
@@ -1558,14 +1558,14 @@ _conn_request_state(struct drbd_tconn *tconn, union drbd_state mask, union drbd_
 	}
 
 	conn_old_common_state(tconn, &os, &flags);
-	ms = conn_set_state(tconn, mask, val, flags);
-	ms.conn = val.conn;
-	conn_pr_state_change(tconn, os, ms, flags);
+	ns_max = conn_set_state(tconn, mask, val, flags);
+	ns_max.conn = val.conn;
+	conn_pr_state_change(tconn, os, ns_max, flags);
 
 	acscw = kmalloc(sizeof(*acscw), GFP_ATOMIC);
 	if (acscw) {
 		acscw->oc = os.conn;
-		acscw->nms = ms;
+		acscw->ns_max = ns_max;
 		acscw->flags = flags;
 		acscw->w.cb = w_after_conn_state_ch;
 		acscw->w.tconn = tconn;
-- 
1.7.4.1

[PATCH 06/11] drbd: Calculate and provide ns_min to the w_after_conn_state_ch() work

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_state.c |   42 ++++++++++++++++++++++++++++----------
 1 files changed, 31 insertions(+), 11 deletions(-)

diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index bd50644..cfd923e 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -1340,6 +1340,7 @@ static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
 struct after_conn_state_chg_work {
 	struct drbd_work w;
 	enum drbd_conns oc;
+	union drbd_state ns_min;
 	union drbd_state ns_max; /* new, max state, over all mdevs */
 	enum chg_state_flags flags;
 };
@@ -1450,11 +1451,17 @@ conn_is_valid_transition(struct drbd_tconn *tconn, union drbd_state mask, union
 	return rv;
 }
 
-static union drbd_state
+void
 conn_set_state(struct drbd_tconn *tconn, union drbd_state mask, union drbd_state val,
-	       enum chg_state_flags flags)
+	       union drbd_state *pns_min, union drbd_state *pns_max, enum chg_state_flags flags)
 {
-	union drbd_state ns, os, ms = { };
+	union drbd_state ns, os, ns_max = { };
+	union drbd_state ns_min = {
+		{ .role = R_MASK,
+		  .peer = R_MASK,
+		  .disk = D_MASK,
+		  .pdsk = D_MASK
+		} };
 	struct drbd_conf *mdev;
 	enum drbd_state_rv rv;
 	int vnr;
@@ -1474,13 +1481,26 @@ conn_set_state(struct drbd_tconn *tconn, union drbd_state mask, union drbd_state
 		if (rv < SS_SUCCESS)
 			BUG();
 
-		ms.role = max_t(enum drbd_role, mdev->state.role, ms.role);
-		ms.peer = max_t(enum drbd_role, mdev->state.peer, ms.peer);
-		ms.disk = max_t(enum drbd_disk_state, mdev->state.disk, ms.disk);
-		ms.pdsk = max_t(enum drbd_disk_state, mdev->state.pdsk, ms.pdsk);
+		ns.i = mdev->state.i;
+		ns_max.role = max_t(enum drbd_role, ns.role, ns_max.role);
+		ns_max.peer = max_t(enum drbd_role, ns.peer, ns_max.peer);
+		ns_max.conn = max_t(enum drbd_role, ns.conn, ns_max.conn);
+		ns_max.disk = max_t(enum drbd_disk_state, ns.disk, ns_max.disk);
+		ns_max.pdsk = max_t(enum drbd_disk_state, ns.pdsk, ns_max.pdsk);
+
+		ns_min.role = min_t(enum drbd_role, ns.role, ns_min.role);
+		ns_min.peer = min_t(enum drbd_role, ns.peer, ns_min.peer);
+		ns_min.conn = min_t(enum drbd_role, ns.conn, ns_min.conn);
+		ns_min.disk = min_t(enum drbd_disk_state, ns.disk, ns_min.disk);
+		ns_min.pdsk = min_t(enum drbd_disk_state, ns.pdsk, ns_min.pdsk);
 	}
 
-	return ms;
+	ns_min.susp = ns_max.susp = tconn->susp;
+	ns_min.susp_nod = ns_max.susp_nod = tconn->susp_nod;
+	ns_min.susp_fen = ns_max.susp_fen = tconn->susp_fen;
+
+	*pns_min = ns_min;
+	*pns_max = ns_max;
 }
 
 static enum drbd_state_rv
@@ -1541,7 +1561,7 @@ _conn_request_state(struct drbd_tconn *tconn, union drbd_state mask, union drbd_
 	enum drbd_state_rv rv = SS_SUCCESS;
 	struct after_conn_state_chg_work *acscw;
 	enum drbd_conns oc = tconn->cstate;
-	union drbd_state ns_max, os;
+	union drbd_state ns_max, ns_min, os;
 
 	rv = is_valid_conn_transition(oc, val.conn);
 	if (rv < SS_SUCCESS)
@@ -1558,13 +1578,13 @@ _conn_request_state(struct drbd_tconn *tconn, union drbd_state mask, union drbd_
 	}
 
 	conn_old_common_state(tconn, &os, &flags);
-	ns_max = conn_set_state(tconn, mask, val, flags);
-	ns_max.conn = val.conn;
+	conn_set_state(tconn, mask, val, &ns_min, &ns_max, flags);
 	conn_pr_state_change(tconn, os, ns_max, flags);
 
 	acscw = kmalloc(sizeof(*acscw), GFP_ATOMIC);
 	if (acscw) {
 		acscw->oc = os.conn;
+		acscw->ns_min = ns_min;
 		acscw->ns_max = ns_max;
 		acscw->flags = flags;
 		acscw->w.cb = w_after_conn_state_ch;
-- 
1.7.4.1

[PATCH 07/11] drbd: Implemented conn_lowest_conn()

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_state.c |   12 ++++++++++++
 drivers/block/drbd/drbd_state.h |    1 +
 2 files changed, 13 insertions(+), 0 deletions(-)

diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index cfd923e..e441d1f 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -115,6 +115,18 @@ enum drbd_disk_state conn_highest_pdsk(struct drbd_tconn *tconn)
 	return ds;
 }
 
+enum drbd_disk_state conn_lowest_conn(struct drbd_tconn *tconn)
+{
+	enum drbd_conns conn = C_MASK;
+	struct drbd_conf *mdev;
+	int vnr;
+
+	idr_for_each_entry(&tconn->volumes, mdev, vnr)
+		conn = min_t(enum drbd_conns, conn, mdev->state.conn);
+
+	return conn;
+}
+
 /**
  * cl_wide_st_chg() - true if the state change is a cluster wide one
  * @mdev:	DRBD device.
diff --git a/drivers/block/drbd/drbd_state.h b/drivers/block/drbd/drbd_state.h
index 0f8441d..e554537 100644
--- a/drivers/block/drbd/drbd_state.h
+++ b/drivers/block/drbd/drbd_state.h
@@ -154,5 +154,6 @@ enum drbd_role conn_highest_role(struct drbd_tconn *tconn);
 enum drbd_role conn_highest_peer(struct drbd_tconn *tconn);
 enum drbd_disk_state conn_highest_disk(struct drbd_tconn *tconn);
 enum drbd_disk_state conn_highest_pdsk(struct drbd_tconn *tconn);
+enum drbd_disk_state conn_lowest_conn(struct drbd_tconn *tconn);
 
 #endif
-- 
1.7.4.1

[PATCH 08/11] drbd: Implemented conn_lowest_disk()

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_state.c |   12 ++++++++++++
 drivers/block/drbd/drbd_state.h |    1 +
 2 files changed, 13 insertions(+), 0 deletions(-)

diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index e441d1f..5db514e 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -103,6 +103,18 @@ enum drbd_disk_state conn_highest_disk(struct drbd_tconn *tconn)
 	return ds;
 }
 
+enum drbd_disk_state conn_lowest_disk(struct drbd_tconn *tconn)
+{
+	enum drbd_disk_state ds = D_MASK;
+	struct drbd_conf *mdev;
+	int vnr;
+
+	idr_for_each_entry(&tconn->volumes, mdev, vnr)
+		ds = min_t(enum drbd_disk_state, ds, mdev->state.disk);
+
+	return ds;
+}
+
 enum drbd_disk_state conn_highest_pdsk(struct drbd_tconn *tconn)
 {
 	enum drbd_disk_state ds = D_DISKLESS;
diff --git a/drivers/block/drbd/drbd_state.h b/drivers/block/drbd/drbd_state.h
index e554537..72e2053 100644
--- a/drivers/block/drbd/drbd_state.h
+++ b/drivers/block/drbd/drbd_state.h
@@ -153,6 +153,7 @@ static inline int drbd_request_state(struct drbd_conf *mdev,
 enum drbd_role conn_highest_role(struct drbd_tconn *tconn);
 enum drbd_role conn_highest_peer(struct drbd_tconn *tconn);
 enum drbd_disk_state conn_highest_disk(struct drbd_tconn *tconn);
+enum drbd_disk_state conn_lowest_disk(struct drbd_tconn *tconn);
 enum drbd_disk_state conn_highest_pdsk(struct drbd_tconn *tconn);
 enum drbd_disk_state conn_lowest_conn(struct drbd_tconn *tconn);
 
-- 
1.7.4.1

[PATCH 09/11] drbd: Implemented IO thawing for multiple volumes

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_state.c |   72 +++++++++++++++++++++-----------------
 1 files changed, 40 insertions(+), 32 deletions(-)

diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index 5db514e..441d80f 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -1070,8 +1070,6 @@ static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
 			   union drbd_state ns, enum chg_state_flags flags)
 {
 	enum drbd_fencing_p fp;
-	enum drbd_req_event what = NOTHING;
-	union drbd_state nsm;
 	struct sib_info sib;
 
 	sib.sib_reason = SIB_STATE_CHANGE;
@@ -1100,44 +1098,21 @@ static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
 	/* Here we have the actions that are performed after a
 	   state change. This function might sleep */
 
-	nsm.i = -1;
 	if (ns.susp_nod) {
-		if (os.conn < C_CONNECTED && ns.conn >= C_CONNECTED)
+		enum drbd_req_event what = NOTHING;
+
+		if (os.conn < C_CONNECTED && conn_lowest_conn(mdev->tconn) >= C_CONNECTED)
 			what = RESEND;
 
-		if (os.disk == D_ATTACHING && ns.disk > D_ATTACHING)
+		if (os.disk == D_ATTACHING && conn_lowest_disk(mdev->tconn) > D_ATTACHING)
 			what = RESTART_FROZEN_DISK_IO;
 
-		if (what != NOTHING)
-			nsm.susp_nod = 0;
-	}
-
-	if (ns.susp_fen) {
-		/* case1: The outdate peer handler is successful: */
-		if (os.pdsk > D_OUTDATED  && ns.pdsk <= D_OUTDATED) {
-			tl_clear(mdev->tconn);
-			if (test_bit(NEW_CUR_UUID, &mdev->flags)) {
-				drbd_uuid_new_current(mdev);
-				clear_bit(NEW_CUR_UUID, &mdev->flags);
-			}
+		if (what != NOTHING) {
 			spin_lock_irq(&mdev->tconn->req_lock);
-			_drbd_set_state(_NS(mdev, susp_fen, 0), CS_VERBOSE, NULL);
+			_tl_restart(mdev->tconn, what);
+			_drbd_set_state(_NS(mdev, susp_nod, 0), CS_VERBOSE, NULL);
 			spin_unlock_irq(&mdev->tconn->req_lock);
 		}
-		/* case2: The connection was established again: */
-		if (os.conn < C_CONNECTED && ns.conn >= C_CONNECTED) {
-			clear_bit(NEW_CUR_UUID, &mdev->flags);
-			what = RESEND;
-			nsm.susp_fen = 0;
-		}
-	}
-
-	if (what != NOTHING) {
-		spin_lock_irq(&mdev->tconn->req_lock);
-		_tl_restart(mdev->tconn, what);
-		nsm.i &= mdev->state.i;
-		_drbd_set_state(mdev, nsm, CS_VERBOSE, NULL);
-		spin_unlock_irq(&mdev->tconn->req_lock);
 	}
 
 	/* Became sync source.  With protocol >= 96, we still need to send out
@@ -1384,6 +1359,9 @@ static int w_after_conn_state_ch(struct drbd_work *w, int unused)
 	struct drbd_tconn *tconn = w->tconn;
 	enum drbd_conns oc = acscw->oc;
 	union drbd_state ns_max = acscw->ns_max;
+	union drbd_state ns_min = acscw->ns_min;
+	struct drbd_conf *mdev;
+	int vnr;
 
 	kfree(acscw);
 
@@ -1391,6 +1369,36 @@ static int w_after_conn_state_ch(struct drbd_work *w, int unused)
 	if (oc == C_STANDALONE && ns_max.conn == C_UNCONNECTED)
 		drbd_thread_start(&tconn->receiver);
 
+	if (ns_max.susp_fen) {
+		/* case1: The outdate peer handler is successful: */
+		if (ns_max.pdsk <= D_OUTDATED) {
+			tl_clear(tconn);
+			idr_for_each_entry(&tconn->volumes, mdev, vnr) {
+				if (test_bit(NEW_CUR_UUID, &mdev->flags)) {
+					drbd_uuid_new_current(mdev);
+					clear_bit(NEW_CUR_UUID, &mdev->flags);
+				}
+			}
+			conn_request_state(tconn,
+					   (union drbd_state) { { .susp_fen = 1 } },
+					   (union drbd_state) { { .susp_fen = 0 } },
+					   CS_VERBOSE);
+		}
+		/* case2: The connection was established again: */
+		if (ns_min.conn >= C_CONNECTED) {
+			idr_for_each_entry(&tconn->volumes, mdev, vnr)
+				clear_bit(NEW_CUR_UUID, &mdev->flags);
+			spin_lock_irq(&tconn->req_lock);
+			_tl_restart(tconn, RESEND);
+			_conn_request_state(tconn,
+					    (union drbd_state) { { .susp_fen = 1 } },
+					    (union drbd_state) { { .susp_fen = 0 } },
+					    CS_VERBOSE);
+			spin_unlock_irq(&tconn->req_lock);
+		}
+	}
+
+
 	//conn_err(tconn, STATE_FMT, STATE_ARGS("nms", nms));
 	after_all_state_ch(tconn);
 
-- 
1.7.4.1

[PATCH 10/11] drbd: Improved logging of state changes

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_state.c |   44 ++++++++++++++++++++++++++------------
 drivers/block/drbd/drbd_state.h |    3 +-
 2 files changed, 32 insertions(+), 15 deletions(-)

diff --git a/drivers/block/drbd/drbd_state.c b/drivers/block/drbd/drbd_state.c
index 441d80f..694bd3f 100644
--- a/drivers/block/drbd/drbd_state.c
+++ b/drivers/block/drbd/drbd_state.c
@@ -366,6 +366,7 @@ static long print_state_change(char *pb, union drbd_state os, union drbd_state n
 	char *pbp;
 	pbp = pb;
 	*pbp = 0;
+
 	if (ns.role != os.role && flags & CS_DC_ROLE)
 		pbp += sprintf(pbp, "role( %s -> %s ) ",
 			       drbd_role_str(os.role),
@@ -386,10 +387,18 @@ static long print_state_change(char *pb, union drbd_state os, union drbd_state n
 		pbp += sprintf(pbp, "pdsk( %s -> %s ) ",
 			       drbd_disk_str(os.pdsk),
 			       drbd_disk_str(ns.pdsk));
-	if (is_susp(ns) != is_susp(os))
-		pbp += sprintf(pbp, "susp( %d -> %d ) ",
-			       is_susp(os),
-			       is_susp(ns));
+
+	return pbp - pb;
+}
+
+static void drbd_pr_state_change(struct drbd_conf *mdev, union drbd_state os, union drbd_state ns,
+				 enum chg_state_flags flags)
+{
+	char pb[300];
+	char *pbp = pb;
+
+	pbp += print_state_change(pbp, os, ns, flags ^ CS_DC_MASK);
+
 	if (ns.aftr_isp != os.aftr_isp)
 		pbp += sprintf(pbp, "aftr_isp( %d -> %d ) ",
 			       os.aftr_isp,
@@ -403,15 +412,7 @@ static long print_state_change(char *pb, union drbd_state os, union drbd_state n
 			       os.user_isp,
 			       ns.user_isp);
 
-	return pbp - pb;
-}
-
-static void drbd_pr_state_change(struct drbd_conf *mdev, union drbd_state os, union drbd_state ns,
-				 enum chg_state_flags flags)
-{
-	char pb[300];
-
-	if (print_state_change(pb, os, ns, flags ^ CS_DC_MASK))
+	if (pbp != pb)
 		dev_info(DEV, "%s\n", pb);
 }
 
@@ -419,8 +420,16 @@ static void conn_pr_state_change(struct drbd_tconn *tconn, union drbd_state os,
 				 enum chg_state_flags flags)
 {
 	char pb[300];
+	char *pbp = pb;
+
+	pbp += print_state_change(pbp, os, ns, flags);
 
-	if (print_state_change(pb, os, ns, flags))
+	if (is_susp(ns) != is_susp(os) && flags & CS_DC_SUSP)
+		pbp += sprintf(pbp, "susp( %d -> %d ) ",
+			       is_susp(os),
+			       is_susp(ns));
+
+	if (pbp != pb)
 		conn_info(tconn, "%s\n", pb);
 }
 
@@ -858,6 +867,12 @@ __drbd_set_state(struct drbd_conf *mdev, union drbd_state ns,
 
 	drbd_pr_state_change(mdev, os, ns, flags);
 
+	/* Display changes to the susp* flags that where caused by the call to
+	   sanitize_state(). Only display it here if we where not called from
+	   _conn_request_state() */
+	if (!(flags & CS_DC_SUSP))
+		conn_pr_state_change(mdev->tconn, os, ns, (flags & ~CS_DC_MASK) | CS_DC_SUSP);
+
 	/* if we are going -> D_FAILED or D_DISKLESS, grab one extra reference
 	 * on the ldev here, to be sure the transition -> D_DISKLESS resp.
 	 * drbd_ldev_destroy() won't happen before our corresponding
@@ -1610,6 +1625,7 @@ _conn_request_state(struct drbd_tconn *tconn, union drbd_state mask, union drbd_
 	}
 
 	conn_old_common_state(tconn, &os, &flags);
+	flags |= CS_DC_SUSP;
 	conn_set_state(tconn, mask, val, &ns_min, &ns_max, flags);
 	conn_pr_state_change(tconn, os, ns_max, flags);
 
diff --git a/drivers/block/drbd/drbd_state.h b/drivers/block/drbd/drbd_state.h
index 72e2053..a49bece 100644
--- a/drivers/block/drbd/drbd_state.h
+++ b/drivers/block/drbd/drbd_state.h
@@ -68,8 +68,9 @@ enum chg_state_flags {
 	CS_DC_CONN       = 1 << 7,
 	CS_DC_DISK       = 1 << 8,
 	CS_DC_PDSK       = 1 << 9,
+	CS_DC_SUSP       = 1 << 10,
 	CS_DC_MASK       = CS_DC_ROLE + CS_DC_PEER + CS_DC_CONN + CS_DC_DISK + CS_DC_PDSK,
-	CS_IGN_OUTD_FAIL = 1 << 10,
+	CS_IGN_OUTD_FAIL = 1 << 11,
 };
 
 /* drbd_dev_state and drbd_state are different types. This is to stress the
-- 
1.7.4.1

[PATCH 11/11] drbd: Use tconn in request_timer_fn()

[Philipp Reisner]

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
---
 drivers/block/drbd/drbd_receiver.c |    1 +
 drivers/block/drbd/drbd_req.c      |   15 ++++++++-------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c
index d6afb3a..6e6b829 100644
--- a/drivers/block/drbd/drbd_receiver.c
+++ b/drivers/block/drbd/drbd_receiver.c
@@ -794,6 +794,7 @@ int drbd_connected(int vnr, void *p, void *data)
 		err = drbd_send_state(mdev);
 	clear_bit(USE_DEGR_WFC_T, &mdev->flags);
 	clear_bit(RESIZE_PENDING, &mdev->flags);
+	mod_timer(&mdev->request_timer, jiffies + HZ); /* just start it here. */
 	return err;
 }
 
diff --git a/drivers/block/drbd/drbd_req.c b/drivers/block/drbd/drbd_req.c
index e296d7e..fd0b452 100644
--- a/drivers/block/drbd/drbd_req.c
+++ b/drivers/block/drbd/drbd_req.c
@@ -1047,21 +1047,22 @@ int drbd_merge_bvec(struct request_queue *q, struct bvec_merge_data *bvm, struct
 void request_timer_fn(unsigned long data)
 {
 	struct drbd_conf *mdev = (struct drbd_conf *) data;
+	struct drbd_tconn *tconn = mdev->tconn;
 	struct drbd_request *req; /* oldest request */
 	struct list_head *le;
 	unsigned long et = 0; /* effective timeout = ko_count * timeout */
 
-	if (get_net_conf(mdev->tconn)) {
-		et = mdev->tconn->net_conf->timeout*HZ/10 * mdev->tconn->net_conf->ko_count;
-		put_net_conf(mdev->tconn);
+	if (get_net_conf(tconn)) {
+		et = tconn->net_conf->timeout*HZ/10 * tconn->net_conf->ko_count;
+		put_net_conf(tconn);
 	}
 	if (!et || mdev->state.conn < C_WF_REPORT_PARAMS)
 		return; /* Recurring timer stopped */
 
-	spin_lock_irq(&mdev->tconn->req_lock);
-	le = &mdev->tconn->oldest_tle->requests;
+	spin_lock_irq(&tconn->req_lock);
+	le = &tconn->oldest_tle->requests;
 	if (list_empty(le)) {
-		spin_unlock_irq(&mdev->tconn->req_lock);
+		spin_unlock_irq(&tconn->req_lock);
 		mod_timer(&mdev->request_timer, jiffies + et);
 		return;
 	}
@@ -1080,5 +1081,5 @@ void request_timer_fn(unsigned long data)
 		mod_timer(&mdev->request_timer, req->start_time + et);
 	}
 
-	spin_unlock_irq(&mdev->tconn->req_lock);
+	spin_unlock_irq(&tconn->req_lock);
 }
-- 
1.7.4.1