[PATCH 1/1] ipc/mqueue: Obey RLIM_INFINITY for message queues.

[Fredrik Markstrom <fredrik.markstrom@gmail.com>]

Even if we set the "POSIX message queues" rlimit to unlimited we fail with
EMFILE if the sum kept per user wraps around after 4G.

This patch fixes that by skipping the test and changing the type of
user->mq_bytes to long long. The accounting wasn't skipped entierly if
RLIM_INFINITY is that someone might change rlimit from unlimited to
something smaller while we have the mq open.

Signed-off-by: Fredrik Markstrom <fredrik.markstrom@gmail.com>
---

/* Compile with: gcc -o mqt mqt.c
 */
#include <fcntl.h>
#include <sys/stat.h>
#include <mqueue.h>
#include <stdio.h>

int main(void) {
	int i;
	mqd_t mqs[1000];
	struct mq_attr attr;

	attr.mq_msgsize = 70000;
	attr.mq_maxmsg = 10000;
	attr.mq_flags = 0;
	attr.mq_curmsgs = 0;

	for(i = 0; i < 1000; i++) {
		char name[32];
		sprintf(name, "/tmq%d", i);
		mqs[i] = mq_open(name, O_RDWR|O_CREAT, 0644, &attr);
		if(mqs[i] < 0) {
			printf("Failed after %d mq_open\n", i);
			perror("mq_open");
			return -1;
		}
	}
	printf("Success (i=%d)\n", i);
	return 0;
}


Before patch:

% ulimit -c unlimited
% ./mqt
Failed after 6 mq_open
mq_open: Too many open files

After patch:

% ulimit -c unlimited
% ./mqt
....
Success (i=1000)


 include/linux/sched.h | 2 +-
 ipc/mqueue.c          | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/include/linux/sched.h b/include/linux/sched.h
index edad7a4..745b7f5 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -827,7 +827,7 @@ struct user_struct {
 #endif
 #ifdef CONFIG_POSIX_MQUEUE
 	/* protected by mq_lock	*/
-	unsigned long mq_bytes;	/* How many bytes can be allocated to mqueue? */
+	unsigned long long mq_bytes;	/* How many bytes can be allocated to mqueue? */
 #endif
 	unsigned long locked_shm; /* How many pages of mlocked shm ? */
 
diff --git a/ipc/mqueue.c b/ipc/mqueue.c
index 161a180..40db042 100644
--- a/ipc/mqueue.c
+++ b/ipc/mqueue.c
@@ -275,8 +275,9 @@ static struct inode *mqueue_get_inode(struct super_block *sb,
 					  info->attr.mq_msgsize);
 
 		spin_lock(&mq_lock);
-		if (u->mq_bytes + mq_bytes < u->mq_bytes ||
-		    u->mq_bytes + mq_bytes > rlimit(RLIMIT_MSGQUEUE)) {
+		if (rlimit(RLIMIT_MSGQUEUE) != RLIM_INFINITY && (
+		    u->mq_bytes + mq_bytes < u->mq_bytes ||
+		    u->mq_bytes + mq_bytes > rlimit(RLIMIT_MSGQUEUE))) {
 			spin_unlock(&mq_lock);
 			/* mqueue_evict_inode() releases info->messages */
 			ret = -EMFILE;
-- 
2.1.4