Commit 487c20b0 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

iov: improve copy_iovec_from_user() code generation 



Use the same pattern as the compat version of this code does: instead of
copying the whole array to a kernel buffer and then having a separate
phase of verifying it, just do it one entry at a time, verifying as you
go.

On Jens' /dev/zero readv() test this improves performance by ~6%.

[ This was obviously triggered by Jens' ITER_UBUF updates series ]

Reported-and-tested-by: default avatarJens Axboe <axboe@kernel.dk>
Link: https://lore.kernel.org/all/de35d11d-bce7-e976-7372-1f2caf417103@kernel.dk/


Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent b9dff219

lib/iov_iter.c

+26 −9
Original line number Diff line number Diff line
@@ -1735,18 +1735,35 @@ static __noclone int copy_compat_iovec_from_user(struct iovec *iov, 
}



static int copy_iovec_from_user(struct iovec *iov,

		const struct iovec __user *uvec, unsigned long nr_segs)

		const struct iovec __user *uiov, unsigned long nr_segs)

{

	unsigned long seg;

	int ret = -EFAULT;



	if (copy_from_user(iov, uvec, nr_segs * sizeof(*uvec)))

	if (!user_access_begin(uiov, nr_segs * sizeof(*uiov)))

		return -EFAULT;

	for (seg = 0; seg < nr_segs; seg++) {

		if ((ssize_t)iov[seg].iov_len < 0)

			return -EINVAL;



	do {

		void __user *buf;

		ssize_t len;



		unsafe_get_user(len, &uiov->iov_len, uaccess_end);

		unsafe_get_user(buf, &uiov->iov_base, uaccess_end);



		/* check for size_t not fitting in ssize_t .. */

		if (unlikely(len < 0)) {

			ret = -EINVAL;

			goto uaccess_end;

		}

		iov->iov_base = buf;

		iov->iov_len = len;



	return 0;

		uiov++; iov++;

	} while (--nr_segs);



	ret = 0;

uaccess_end:

	user_access_end();

	return ret;

}



struct iovec *iovec_from_user(const struct iovec __user *uvec,
@@ -1771,7 +1788,7 @@ struct iovec *iovec_from_user(const struct iovec __user *uvec, 
			return ERR_PTR(-ENOMEM);

	}



	if (compat)

	if (unlikely(compat))

		ret = copy_compat_iovec_from_user(iov, uvec, nr_segs);

	else

		ret = copy_iovec_from_user(iov, uvec, nr_segs);