additional hash for local port; improves speed of implicit bind

from >1000K cpu cycles to 20-30K for 18000 sockets on i386;
test+feedback by Claudio Jeker; ok itojun@;
[make sure you rebuild netstat/systat, too]

2 files changed, 20 insertions, 8 deletions

diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
index 3136ac4366d..0ceb29fa4e6 100644
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: in_pcb.c,v 1.67 2003/08/15 20:32:20 tedu Exp $	*/
+/*	$OpenBSD: in_pcb.c,v 1.68 2003/10/25 12:15:24 markus Exp $	*/
 /*	$NetBSD: in_pcb.c,v 1.25 1996/02/13 23:41:53 christos Exp $	*/
 
 /*
@@ -117,6 +117,9 @@ int ipport_hilastauto = IPPORT_HILASTAUTO;	/* 44999 */
 	(faddr)->s6_addr32[3]) + ntohs((fport)) + ntohs((lport))) & \
 	(table->inpt_hash)]
 
+#define	INPCBLHASH(table, lport) \
+	&(table)->inpt_lhashtbl[lport & table->inpt_lhash]
+
 void
 in_pcbinit(table, hashsize)
 	struct inpcbtable *table;
@@ -124,9 +127,14 @@ in_pcbinit(table, hashsize)
 {
 
 	CIRCLEQ_INIT(&table->inpt_queue);
-	table->inpt_hashtbl = hashinit(hashsize, M_PCB, M_NOWAIT, &table->inpt_hash);
+	table->inpt_hashtbl = hashinit(hashsize, M_PCB, M_NOWAIT,
+	    &table->inpt_hash);
 	if (table->inpt_hashtbl == NULL)
 		panic("in_pcbinit: hashinit failed");
+	table->inpt_lhashtbl = hashinit(hashsize, M_PCB, M_NOWAIT,
+	    &table->inpt_lhash);
+	if (table->inpt_lhashtbl == NULL)
+		panic("in_pcbinit: hashinit failed for lport");
 	table->inpt_lastport = 0;
 }
 
@@ -175,6 +183,7 @@ in_pcballoc(so, v)
 	inp->inp_seclevel[SL_IPCOMP] = ipsec_ipcomp_default_level;
 	s = splnet();
 	CIRCLEQ_INSERT_HEAD(&table->inpt_queue, inp, inp_queue);
+	LIST_INSERT_HEAD(INPCBLHASH(table, inp->inp_lport), inp, inp_lhash);
 	LIST_INSERT_HEAD(INPCBHASH(table, &inp->inp_faddr, inp->inp_fport,
 	    &inp->inp_laddr, inp->inp_lport), inp, inp_hash);
 	splx(s);
@@ -496,6 +505,7 @@ in_pcbdetach(v)
 	splx(s);
 #endif
 	s = splnet();
+	LIST_REMOVE(inp, inp_lhash);
 	LIST_REMOVE(inp, inp_hash);
 	CIRCLEQ_REMOVE(&inp->inp_table->inpt_queue, inp, inp_queue);
 	splx(s);
@@ -713,9 +723,8 @@ in_pcblookup(table, faddrp, fport_arg, laddrp, lport_arg, flags)
 	struct in_addr faddr = *(struct in_addr *)faddrp;
 	struct in_addr laddr = *(struct in_addr *)laddrp;
 
-	for (inp = table->inpt_queue.cqh_first;
-	    inp != (struct inpcb *)&table->inpt_queue;
-	    inp = inp->inp_queue.cqe_next) {
+	for (inp = LIST_FIRST(INPCBLHASH(table, lport)); inp;
+	    inp = LIST_NEXT(inp, inp_lhash)) {
 		if (inp->inp_lport != lport)
 			continue;
 		wildcard = 0;
@@ -923,6 +932,8 @@ in_pcbrehash(inp)
 	int s;
 
 	s = splnet();
+	LIST_REMOVE(inp, inp_lhash);
+	LIST_INSERT_HEAD(INPCBLHASH(table, inp->inp_lport), inp, inp_lhash);
 	LIST_REMOVE(inp, inp_hash);
 #ifdef INET6
 	if (inp->inp_flags & INP_IPV6) {
diff --git a/sys/netinet/in_pcb.h b/sys/netinet/in_pcb.h
index 015668a0b9e..684c941b634 100644
--- a/sys/netinet/in_pcb.h
+++ b/sys/netinet/in_pcb.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: in_pcb.h,v 1.45 2003/06/02 23:28:14 millert Exp $	*/
+/*	$OpenBSD: in_pcb.h,v 1.46 2003/10/25 12:15:24 markus Exp $	*/
 /*	$NetBSD: in_pcb.h,v 1.14 1996/02/13 23:42:00 christos Exp $	*/
 
 /*
@@ -87,6 +87,7 @@ union inpaddru {
  */
 struct inpcb {
 	LIST_ENTRY(inpcb) inp_hash;
+	LIST_ENTRY(inpcb) inp_lhash;		/* extra hash for lport */
 	CIRCLEQ_ENTRY(inpcb) inp_queue;
 	struct	  inpcbtable *inp_table;
 	union	  inpaddru inp_faddru;		/* Foreign address. */
@@ -147,8 +148,8 @@ struct inpcb {
 
 struct inpcbtable {
 	CIRCLEQ_HEAD(, inpcb) inpt_queue;
-	LIST_HEAD(inpcbhead, inpcb) *inpt_hashtbl;
-	u_long	  inpt_hash;
+	LIST_HEAD(inpcbhead, inpcb) *inpt_hashtbl, *inpt_lhashtbl;
+	u_long	  inpt_hash, inpt_lhash;
 	u_int16_t inpt_lastport;
 };