[PATCH 1/1] ipc/mqueue: Obey RLIM_INFINITY for message queues.

[Fredrik Markstrom <fredrik.markstrom@gmail.com>]

Even if we set the "POSIX message queues" rlimit to unlimited we might
fail with EMFILE. That happens when the max usage of a user wraps around
on 32 bits.

We fix this by:
- Skipping the test in the case of RLIM_INFINITY
- Changing user->mq_bytes from long to long long

The accounting can't be skipped entierly for this case since rlimit can be
changed from unlimited to something smaller while a message queue is open.

Signed-off-by: Fredrik Markstrom <fredrik.markstrom@gmail.com>
---

/* Compile with: gcc -o mqt mqt.c
 */
#include <fcntl.h>
#include <sys/stat.h>
#include <mqueue.h>
#include <stdio.h>

int main(void) {
	int i;
	mqd_t mqs[1000];
	struct mq_attr attr;

	attr.mq_msgsize = 70000;
	attr.mq_maxmsg = 10000;
	attr.mq_flags = 0;
	attr.mq_curmsgs = 0;

	for(i = 0; i < 1000; i++) {
		char name[32];
		sprintf(name, "/tmq%d", i);
		mqs[i] = mq_open(name, O_RDWR|O_CREAT, 0644, &attr);
		if(mqs[i] < 0) {
			printf("Failed after %d mq_open\n", i);
			perror("mq_open");
			return -1;
		}
	}
	printf("Success (i=%d)\n", i);
	return 0;
}


Before patch:

% ulimit -c unlimited
% ./mqt
Failed after 6 mq_open
mq_open: Too many open files

After patch:

% ulimit -c unlimited
% ./mqt
....
Success (i=1000)


 include/linux/sched.h | 2 +-
 ipc/mqueue.c          | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/include/linux/sched.h b/include/linux/sched.h
index edad7a4..745b7f5 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -827,7 +827,7 @@ struct user_struct {
 #endif
 #ifdef CONFIG_POSIX_MQUEUE
 	/* protected by mq_lock	*/
-	unsigned long mq_bytes;	/* How many bytes can be allocated to mqueue? */
+	unsigned long long mq_bytes;	/* How many bytes can be allocated to mqueue? */
 #endif
 	unsigned long locked_shm; /* How many pages of mlocked shm ? */
 
diff --git a/ipc/mqueue.c b/ipc/mqueue.c
index 161a180..40db042 100644
--- a/ipc/mqueue.c
+++ b/ipc/mqueue.c
@@ -275,8 +275,9 @@ static struct inode *mqueue_get_inode(struct super_block *sb,
 					  info->attr.mq_msgsize);
 
 		spin_lock(&mq_lock);
-		if (u->mq_bytes + mq_bytes < u->mq_bytes ||
-		    u->mq_bytes + mq_bytes > rlimit(RLIMIT_MSGQUEUE)) {
+		if (rlimit(RLIMIT_MSGQUEUE) != RLIM_INFINITY && (
+		    u->mq_bytes + mq_bytes < u->mq_bytes ||
+		    u->mq_bytes + mq_bytes > rlimit(RLIMIT_MSGQUEUE))) {
 			spin_unlock(&mq_lock);
 			/* mqueue_evict_inode() releases info->messages */
 			ret = -EMFILE;
-- 
2.1.4