But aren't credential changes supposed to be infrequent?

If so, isn't it faster to put the fields directly in task_struct, keep a
separate shared structure and copy them on kernel entry?

Here is some pseudocode to better illustrate the idea.

vfs_shared_cred::lock can be removed if it's not necessary to do atomic
modifications of multiple cred fields (otherwise you could copy and then
exchange a pointer).

vfs_shared_cred::tasklist_lock can be replaced with RCU.

struct vfs_cred_groups
{
	int ngroups;
	gid_t groups[];
};

struct vfs_cred
{
	uid_t uid, gid;
	struct vfs_cred_groups* groups;
};

struct vfs_shared_cred
{
	atomic_t count;
	spinlock_t lock;
	spinlock_t tasklist_lock;
	vfs_cred cred;
};

struct task_struct
{
	struct vfs_cred cred;
	struct vfs_cred* shared_cred;
	list_t shared_cred_list;
};

struct thread_info
{
	unsigned propagate_cred;
};

propagate_cred()
{
	current_thread_info()->propagate_cred = 0;

	spin_lock(&current->shared_cred->lock);
	current->cred = current->shared_cred->cred;
	spin_unlock(&current->shared_cred->lock);
}

kernel_entry() /* asm for every architecture :( */
{
	if(unlikely(current_thread_info()->propagate_cred))
		propagate_cred();
}

change_credentials()
{
	list_t list;

	spin_lock(&current->shared_cred->lock);
	frob(current->shared_cred);
	spin_unlock(&current->shared_cred->lock);

	wmb();

	spin_lock(&current->shared_cred->tasklist_lock);
	list_for_each(list, &current->shared_cred_list)
	{
		struct task_struct* task = list_entry(list, struct task_struct, shared_cred_list);
		task->thread_info->propagate_cred = 1;
	}
	spin_unlock(&current->shared_cred->tasklist_lock);
}

clone()
{
	spin_lock(&current->shared_cred->lock);
	new->cred = current->shared_cred->cred); /* not current->cred! */
	spin_unlock(&current->shared_cred->lock);

	if(flags & CLONE_CRED)
	{
		new->shared_cred = get_shared_cred(current->shared_cred);
		spin_lock(&current->shared_cred->tasklist_lock);
		list_add(new, &current->shared_cred_list);
		spin_unlock(&current->shared_cred->tasklist_lock);
	}
}

static void release_task(struct task_struct * p)
{
	spin_lock(&current->shared_cred->tasklist_lock);
	__list_del(current->shared_cred_list);
	spin_unlock(&current->shared_cred->tasklist_lock);

	put_shared_cred(current->shared_cred);
}