Protect per-file counters and document which lock is used to protect

the other fields.

Once we no longer have any [k] (kernel lock) protections, we'll be
able to unlock almost all network related syscalls.

Inputs from and ok bluhm@, visa@

6 files changed, 40 insertions, 20 deletions

diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index c868f0aae4f..fc971435284 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_descrip.c,v 1.156 2018/05/02 02:24:56 visa Exp $	*/
+/*	$OpenBSD: kern_descrip.c,v 1.157 2018/05/08 08:53:41 mpi Exp $	*/
 /*	$NetBSD: kern_descrip.c,v 1.42 1996/03/30 22:24:38 christos Exp $	*/
 
 /*
@@ -957,6 +957,7 @@ restart:
 	 */
 	numfiles++;
 	fp = pool_get(&file_pool, PR_WAITOK|PR_ZERO);
+	mtx_init(&fp->f_mtx, IPL_NONE);
 	fp->f_iflags = FIF_LARVAL;
 	if ((fq = p->p_fd->fd_ofiles[0]) != NULL) {
 		LIST_INSERT_AFTER(fq, fp, f_list);
diff --git a/sys/kern/kern_sysctl.c b/sys/kern/kern_sysctl.c
index 8299f50be6a..3ac0efa3fdf 100644
--- a/sys/kern/kern_sysctl.c
+++ b/sys/kern/kern_sysctl.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sysctl.c,v 1.333 2018/04/25 10:29:16 mpi Exp $	*/
+/*	$OpenBSD: kern_sysctl.c,v 1.334 2018/05/08 08:53:41 mpi Exp $	*/
 /*	$NetBSD: kern_sysctl.c,v 1.17 1996/05/20 17:49:05 mrg Exp $	*/
 
 /*-
@@ -1067,11 +1067,13 @@ fill_file(struct kinfo_file *kf, struct file *fp, struct filedesc *fdp,
 
 		if (suser(p) == 0 || p->p_ucred->cr_uid == fp->f_cred->cr_uid) {
 			kf->f_offset = fp->f_offset;
+			mtx_enter(&fp->f_mtx);
 			kf->f_rxfer = fp->f_rxfer;
 			kf->f_rwfer = fp->f_wxfer;
 			kf->f_seek = fp->f_seek;
 			kf->f_rbytes = fp->f_rbytes;
 			kf->f_wbytes = fp->f_wbytes;
+			mtx_leave(&fp->f_mtx);
 		} else
 			kf->f_offset = -1;
 	} else if (vp != NULL) {
diff --git a/sys/kern/sys_generic.c b/sys/kern/sys_generic.c
index 2bcf1804ec0..f8c1635af48 100644
--- a/sys/kern/sys_generic.c
+++ b/sys/kern/sys_generic.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sys_generic.c,v 1.118 2018/04/27 10:13:37 mpi Exp $	*/
+/*	$OpenBSD: sys_generic.c,v 1.119 2018/05/08 08:53:41 mpi Exp $	*/
 /*	$NetBSD: sys_generic.c,v 1.24 1996/03/29 00:25:32 cgd Exp $	*/
 
 /*
@@ -203,8 +203,10 @@ dofilereadv(struct proc *p, int fd, struct file *fp, const struct iovec *iovp,
 			error = 0;
 	cnt -= auio.uio_resid;
 
+	mtx_enter(&fp->f_mtx);
 	fp->f_rxfer++;
 	fp->f_rbytes += cnt;
+	mtx_leave(&fp->f_mtx);
 #ifdef KTRACE
 	if (ktriov != NULL) {
 		if (error == 0)
@@ -355,8 +357,10 @@ dofilewritev(struct proc *p, int fd, struct file *fp, const struct iovec *iovp,
 	}
 	cnt -= auio.uio_resid;
 
+	mtx_enter(&fp->f_mtx);
 	fp->f_wxfer++;
 	fp->f_wbytes += cnt;
+	mtx_leave(&fp->f_mtx);
 #ifdef KTRACE
 	if (ktriov != NULL) {
 		if (error == 0)
diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index ed63535d9e2..483f803a0df 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uipc_syscalls.c,v 1.169 2018/04/27 10:13:37 mpi Exp $	*/
+/*	$OpenBSD: uipc_syscalls.c,v 1.170 2018/05/08 08:53:41 mpi Exp $	*/
 /*	$NetBSD: uipc_syscalls.c,v 1.19 1996/02/09 19:00:48 christos Exp $	*/
 
 /*
@@ -687,8 +687,10 @@ sendit(struct proc *p, int s, struct msghdr *mp, int flags, register_t *retsize)
 	}
 	if (error == 0) {
 		*retsize = len - auio.uio_resid;
+		mtx_enter(&fp->f_mtx);
 		fp->f_wxfer++;
 		fp->f_wbytes += *retsize;
+		mtx_leave(&fp->f_mtx);
 	}
 #ifdef KTRACE
 	if (ktriov != NULL) {
@@ -902,8 +904,10 @@ recvit(struct proc *p, int s, struct msghdr *mp, caddr_t namelenp,
 		mp->msg_controllen = len;
 	}
 	if (!error) {
+		mtx_enter(&fp->f_mtx);
 		fp->f_rxfer++;
 		fp->f_rbytes += *retsize;
+		mtx_leave(&fp->f_mtx);
 	}
 out:
 	FRELE(fp, p);
diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c
index e95b7a59a81..a39376f6fe7 100644
--- a/sys/kern/vfs_syscalls.c
+++ b/sys/kern/vfs_syscalls.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vfs_syscalls.c,v 1.282 2018/05/02 02:24:56 visa Exp $	*/
+/*	$OpenBSD: vfs_syscalls.c,v 1.283 2018/05/08 08:53:41 mpi Exp $	*/
 /*	$NetBSD: vfs_syscalls.c,v 1.71 1996/04/23 10:29:02 mycroft Exp $	*/
 
 /*
@@ -1650,7 +1650,9 @@ sys_lseek(struct proc *p, void *v, register_t *retval)
 		}
 	}
 	*(off_t *)retval = fp->f_offset = newoff;
+	mtx_enter(&fp->f_mtx);
 	fp->f_seek++;
+	mtx_leave(&fp->f_mtx);
 	error = 0;
  bad:
 	FRELE(fp, p);
diff --git a/sys/sys/file.h b/sys/sys/file.h
index 62985415f8f..15438d4c4ee 100644
--- a/sys/sys/file.h
+++ b/sys/sys/file.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: file.h,v 1.41 2018/04/25 10:29:16 mpi Exp $	*/
+/*	$OpenBSD: file.h,v 1.42 2018/05/08 08:53:41 mpi Exp $	*/
 /*	$NetBSD: file.h,v 1.11 1995/03/26 20:24:13 jtc Exp $	*/
 
 /*
@@ -37,6 +37,7 @@
 
 #else /* _KERNEL */
 #include <sys/queue.h>
+#include <sys/mutex.h>
 
 struct proc;
 struct uio;
@@ -61,26 +62,32 @@ struct	fileops {
 /*
  * Kernel descriptor table.
  * One entry for each open kernel vnode and socket.
+ *
+ *  Locks used to protect struct members in this file:
+ *	I	immutable after creation
+ *	f	per file `f_mtx'
+ *	k	kernel lock
  */
 struct file {
-	LIST_ENTRY(file) f_list;/* list of active files */
-	short	f_flag;		/* see fcntl.h */
+	LIST_ENTRY(file) f_list;/* [k] list of active files */
+	struct mutex f_mtx;
+	short	f_flag;		/* [k] see fcntl.h */
 #define	DTYPE_VNODE	1	/* file */
 #define	DTYPE_SOCKET	2	/* communications endpoint */
 #define	DTYPE_PIPE	3	/* pipe */
 #define	DTYPE_KQUEUE	4	/* event queue */
-	short	f_type;		/* descriptor type */
-	long	f_count;	/* reference count */
-	struct	ucred *f_cred;	/* credentials associated with descriptor */
-	struct	fileops *f_ops;
-	off_t	f_offset;
-	void 	*f_data;	/* private data */
-	int	f_iflags;	/* internal flags */
-	u_int64_t f_rxfer;	/* total number of read transfers */
-	u_int64_t f_wxfer;	/* total number of write transfers */
-	u_int64_t f_seek;	/* total independent seek operations */
-	u_int64_t f_rbytes;	/* total bytes read */
-	u_int64_t f_wbytes;	/* total bytes written */
+	short	f_type;		/* [I] descriptor type */
+	long	f_count;	/* [k] reference count */
+	struct	ucred *f_cred;	/* [I] credentials associated with descriptor */
+	struct	fileops *f_ops; /* [k] file operation pointers */
+	off_t	f_offset;	/* [k] */
+	void 	*f_data;	/* [k] private data */
+	int	f_iflags;	/* [k] internal flags */
+	uint64_t f_rxfer;	/* [f] total number of read transfers */
+	uint64_t f_wxfer;	/* [f] total number of write transfers */
+	uint64_t f_seek;	/* [f] total independent seek operations */
+	uint64_t f_rbytes;	/* [f] total bytes read */
+	uint64_t f_wbytes;	/* [f] total bytes written */
 };
 
 #define FIF_HASLOCK		0x01	/* descriptor holds advisory lock */