extend the url lookup algorithm to match the full URL and different

possible suffix/prefix combinations by stripping subdomains, path
components, and the query args.

ok and tested by gilles@

4 files changed, 146 insertions, 20 deletions

diff --git a/usr.sbin/relayd/relay.c b/usr.sbin/relayd/relay.c
index 20550aaef0f..9333d93deed 100644
--- a/usr.sbin/relayd/relay.c
+++ b/usr.sbin/relayd/relay.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relay.c,v 1.68 2007/11/24 13:39:24 reyk Exp $	*/
+/*	$OpenBSD: relay.c,v 1.69 2007/11/24 16:13:50 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006, 2007 Reyk Floeter <reyk@openbsd.org>
@@ -91,6 +91,8 @@ int		 relay_handle_http(struct ctl_relay_event *,
 		    struct protonode *, struct protonode *,
 		    struct protonode *, int);
 void		 relay_read_http(struct bufferevent *, void *);
+static int	_relay_lookup_url(struct ctl_relay_event *, char *, char *,
+		    char *, enum digest_type);
 int		 relay_lookup_url(struct ctl_relay_event *,
 		    const char *, enum digest_type);
 int		 relay_lookup_query(struct ctl_relay_event *);
@@ -383,18 +385,22 @@ relay_protodebug(struct relay *rlay)
 	name = "request";
 	tree = &proto->request_tree;
  show:
+	i = 0;
 	RB_FOREACH(proot, proto_tree, tree) {
-		i = 0;
 		PROTONODE_FOREACH(pn, proot, entry) {
 #ifndef DEBUG
-			/* Limit the number of displayed lines */
-			if (++i > 100) {
-				fprintf(stderr, "\t\t...\n");
+			if (++i > 100)
 				break;
-			}
 #endif
 			relay_nodedebug(name, pn);
 		}
+#ifndef DEBUG
+		/* Limit the number of displayed lines */
+		if (++i > 100) {
+			fprintf(stderr, "\t\t...\n");
+			break;
+		}
+#endif
 	}
 	if (tree == &proto->request_tree) {
 		name = "response";
@@ -1522,26 +1528,25 @@ relay_read_http(struct bufferevent *bev, void *arg)
 	free(line);
 }
 
-int
-relay_lookup_url(struct ctl_relay_event *cre, const char *str,
-    enum digest_type type)
+static int
+_relay_lookup_url(struct ctl_relay_event *cre, char *host, char *path,
+    char *query, enum digest_type type)
 {
 	struct session		*con = (struct session *)cre->con;
 	struct protonode	*proot, *pnv, pkv;
 	char			*val, *md = NULL;
 	int			 ret = PN_FAIL;
 
-	if (cre->path == NULL)
-		return (PN_PASS);
-
 	if (asprintf(&val, "%s%s%s%s",
-	    str, cre->path,
-	    cre->args == NULL ? "" : "?",
-	    cre->args == NULL ? "" : cre->args) == -1) {
+	    host, path,
+	    query == NULL ? "" : "?",
+	    query == NULL ? "" : query) == -1) {
 		relay_close_http(con, 500, "failed to allocate URL");
 		return (PN_FAIL);
 	}
 
+	DPRINTF("_relay_lookup_url: %s", val);
+
 	switch (type) {
 	case DIGEST_SHA1:
 	case DIGEST_MD5:
@@ -1577,6 +1582,82 @@ relay_lookup_url(struct ctl_relay_event *cre, const char *str,
 }
 
 int
+relay_lookup_url(struct ctl_relay_event *cre, const char *str,
+    enum digest_type type)
+{
+	struct session	*con = (struct session *)cre->con;
+	int		 i, j, dots;
+	char		*hi[RELAY_MAXLOOKUPLEVELS], *p, *pp, *c, ch;
+	char		 ph[MAXHOSTNAMELEN];
+	int		 ret;
+
+	if (cre->path == NULL)
+		return (PN_PASS);
+
+	/*
+	 * This is an URL lookup algorithm inspired by
+	 * http://code.google.com/apis/safebrowsing/
+	 *     developers_guide.html#PerformingLookups
+	 */
+
+	DPRINTF("relay_lookup_url: host: '%s', path: '%s', query: '%s'",
+	    str, cre->path, cre->args == NULL ? "" : cre->args);
+
+	if (canonicalize_host(str, ph, sizeof(ph)) == NULL) {
+		relay_close_http(con, 400, "invalid host name");
+		return (PN_FAIL);
+	}
+
+	bzero(hi, sizeof(hi));
+	for (dots = -1, i = strlen(ph) - 1; i > 0; i--) {
+		if (ph[i] == '.' && ++dots)
+			hi[dots - 1] = &ph[i + 1];
+		if (dots > (RELAY_MAXLOOKUPLEVELS - 2))
+			break;
+	}
+	hi[dots] = ph;
+
+	if ((pp = strdup(cre->path)) == NULL) {
+		relay_close_http(con, 500, "failed to allocate path");
+		return (PN_FAIL);
+	}
+	for (i = (RELAY_MAXLOOKUPLEVELS - 1); i >= 0; i--) {
+		if (hi[i] == NULL)
+			continue;
+
+		/* 1. complete path with query */
+		if (cre->args != NULL)
+			if ((ret = _relay_lookup_url(cre, hi[i],
+			    pp, cre->args, type)) != PN_PASS)
+				goto done;
+
+		/* 2. complete path without query */
+		if ((ret = _relay_lookup_url(cre, hi[i],
+		    pp, NULL, type)) != PN_PASS)
+			goto done;
+
+		/* 3. traverse path */
+		for (j = 0, p = strchr(pp, '/');
+		    p != NULL; p = strchr(p, '/'), j++) {
+			if (j > (RELAY_MAXLOOKUPLEVELS - 2) || ++p == '\0')
+				break;
+			c = &pp[p - pp];
+			ch = *c;
+			*c = '\0';
+			if ((ret = _relay_lookup_url(cre, hi[i],
+			    pp, NULL, type)) != PN_PASS)
+				goto done;
+			*c = ch;
+		}
+	}
+
+	ret = PN_PASS;
+ done:
+	free(pp);
+	return (ret);
+}
+
+int
 relay_lookup_query(struct ctl_relay_event *cre)
 {
 	struct session		*con = (struct session *)cre->con;
diff --git a/usr.sbin/relayd/relayd.c b/usr.sbin/relayd/relayd.c
index f0ea86ef005..9e05a2d44cb 100644
--- a/usr.sbin/relayd/relayd.c
+++ b/usr.sbin/relayd/relayd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayd.c,v 1.57 2007/11/23 09:22:18 sthen Exp $	*/
+/*	$OpenBSD: relayd.c,v 1.58 2007/11/24 16:13:50 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -22,6 +22,8 @@
 #include <sys/param.h>
 #include <sys/wait.h>
 #include <net/if.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
 #include <string.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -910,3 +912,23 @@ digeststr(enum digest_type type, const u_int8_t *data, size_t len, char *buf)
 	}
 	return (NULL);
 }
+
+const char *
+canonicalize_host(const char *host, char *name, size_t len)
+{
+	struct sockaddr_in	 sin4;
+	struct sockaddr_in6	 sin6;
+
+	if (inet_pton(AF_INET, host, &sin4) == 1)
+		return (inet_ntop(AF_INET, &sin4, name, len));
+	if (inet_pton(AF_INET6, host, &sin6) == 1)
+		return (inet_ntop(AF_INET6, &sin6, name, len));
+
+	/* XXX canonicalize a FQDN... */
+	if (strlcpy(name, host, len) >= len) {
+		errno = EINVAL;
+		return (NULL);
+	}
+
+	return (name);
+}
diff --git a/usr.sbin/relayd/relayd.conf.5 b/usr.sbin/relayd/relayd.conf.5
index 91ecd9f6253..e6155c5f00b 100644
--- a/usr.sbin/relayd/relayd.conf.5
+++ b/usr.sbin/relayd/relayd.conf.5
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: relayd.conf.5,v 1.65 2007/11/23 09:45:33 reyk Exp $
+.\"	$OpenBSD: relayd.conf.5,v 1.66 2007/11/24 16:13:50 reyk Exp $
 .\"
 .\" Copyright (c) 2006, 2007 Reyk Floeter <reyk@openbsd.org>
 .\" Copyright (c) 2006, 2007 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -15,7 +15,7 @@
 .\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 .\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 .\"
-.Dd $Mdocdate: November 23 2007 $
+.Dd $Mdocdate: November 24 2007 $
 .Dt HOSTSTATED.CONF 5
 .Os
 .Sh NAME
@@ -574,7 +574,7 @@ request query expect "bar" from "foo"
 .Ed
 .It Ic url
 Look up the entity as a URL suffix/prefix expression consisting of a
-hostname or suffix and a path name or prefix when using the
+hostname without port or suffix and a path name or prefix when using the
 .Ic http
 protocol.
 This type is only available with the direction
@@ -584,6 +584,27 @@ for example:
 request url filter "example.com/index.html"
 request url filter "example.com/test.cgi?val=1"
 .Ed
+.Pp
+.Xr hoststated 8
+will match the full URL and different possible suffix/prefix
+combinations by stripping subdomains and path components (up to 5
+levels), and the query string. For example, the following
+lookups will be done for
+.Ar http://www.example.com:81/1/2/3/4/5.html?query=yes :
+.Bd -literal -offset indent
+www.example.com/1/2/3/4/5.html?query=yes
+www.example.com/1/2/3/4/5.html
+www.example.com/
+www.example.com/1/
+www.example.com/1/2/
+www.example.com/1/2/3/
+example.com/1/2/3/4/5.html?query=yes
+example.com/1/2/3/4/5.html
+example.com/
+example.com/1/
+example.com/1/2/
+example.com/1/2/3/
+.Ed
 .El
 .Pp
 The following actions are available:
diff --git a/usr.sbin/relayd/relayd.h b/usr.sbin/relayd/relayd.h
index faba45d4399..daeb30d8776 100644
--- a/usr.sbin/relayd/relayd.h
+++ b/usr.sbin/relayd/relayd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: relayd.h,v 1.84 2007/11/23 09:39:42 reyk Exp $	*/
+/*	$OpenBSD: relayd.h,v 1.85 2007/11/24 16:13:50 reyk Exp $	*/
 
 /*
  * Copyright (c) 2006, 2007 Pierre-Yves Ritschard <pyr@openbsd.org>
@@ -44,6 +44,7 @@
 #define RELAY_MAXHOSTS		32
 #define RELAY_STATINTERVAL	60
 #define RELAY_BACKLOG		10
+#define RELAY_MAXLOOKUPLEVELS	5
 
 #define SMALL_READ_BUF_SIZE	1024
 #define READ_BUF_SIZE		65535
@@ -791,6 +792,7 @@ void		 translate_string(char *);
 void		 purge_config(struct hoststated *, u_int8_t);
 void		 merge_config(struct hoststated *, struct hoststated *);
 char		*digeststr(enum digest_type, const u_int8_t *, size_t, char *);
+const char	*canonicalize_host(const char *, char *, size_t);
 
 /* carp.c */
 int	 carp_demote_init(char *, int);