New spamd configuration method. Many people have trouble with the spam

RBL sites being slow, so now we will provide the maps ourselves through
our www mirrors around the world.  We can also now write our own internal
translators for maps that are in bad formats, and place them into the www
space in the correct format.  tested by beck, djm

4 files changed, 76 insertions, 21 deletions

diff --git a/etc/spamd.conf b/etc/spamd.conf
index 5c279a34a38..c4e719ce090 100644
--- a/etc/spamd.conf
+++ b/etc/spamd.conf
@@ -1,4 +1,4 @@
-# $OpenBSD: spamd.conf,v 1.8 2003/12/04 18:59:29 deraadt Exp $
+# $OpenBSD: spamd.conf,v 1.9 2004/01/21 08:07:39 deraadt Exp $
 #
 # spamd config file, read by spamd-setup(8) for spamd(8)
 #
@@ -17,44 +17,54 @@
 # As of Aug 2003, a place to search for black lists is
 #     http://spamlinks.port5.com/filter-bl.htm#ip
 #
+# Some of the URLs below point to www.openbsd.org locations.  Those
+# files are likely to be mirrored to other OpenBSD www mirrors located
+# around the world.  Hence, it is possible to edit this file and rewrite
+# www.openbsd.org with, for instance, to www.de.openbsd.org
 
 all:\
 	:spamhaus:china:korea:
 
+# Mirrored from http://spfilter.openrbl.org/data/sbl/SBL.cidr.bz2
 spamhaus:\
 	:black:\
 	:msg="SPAM. Your address %A is in the Spamhaus Block List\n\
-	See http://www.spamhaus.org/sbl for more details":\
+	See http://www.spamhaus.org/sbl and\
+	http://www.abuse.net/sbl.phtml?IP=%A for more details":\
 	:method=http:\
-	:file=spfilter.openrbl.org/data/sbl/SBL.cidr
+	:file=www.openbsd.org/spamd/SBL.cidr.gz
 
+# Mirrored from http://www.spews.org/spews_list_level1.txt
 spews1:\
 	:black:\
 	:msg="SPAM. Your address %A is in the spews level 1 database\n\
 	See http://www.spews.org/ask.cgi?x=%A for more details":\
 	:method=http:\
-	:file=www.spews.org/spews_list_level1.txt:
+	:file=www.openbsd.org/spamd/spews_list_level1.txt.gz
 
+# Mirrored from http://www.spews.org/spews_list_level2.txt
 spews2:\
 	:black:\
 	:msg="SPAM. Your address %A is in the spews level 2 database\n\
 	See http://www.spews.org/ask.cgi?x=%A for more details":\
 	:method=http:\
-	:file=www.spews.org/spews_list_level2.txt:
+	:file=www.openbsd.org/spamd/spews_list_level2.txt.gz
 
+# Mirrored from http://www.okean.com/chinacidr.txt
 china:\
 	:black:\
 	:msg="SPAM. Your address %A appears to be from China\n\
 	See http://www.okean.com/asianspamblocks.html for more details":\
 	:method=http:\
-	:file=www.okean.com/chinacidr.txt:
+	:file=www.openbsd.org/spamd/chinacidr.txt.gz
 
+# Mirrored from http://www.okean.com/koreacidr.txt
 korea:\
 	:black:\
 	:msg="SPAM. Your address %A appears to be from Korea\n\
 	See http://www.okean.com/asianspamblocks.html for more details":\
 	:method=http:\
-	:file=www.okean.com/koreacidr.txt:
+	:file=www.openbsd.org/spamd/koreacidr.txt.gz
 
 
 # Whitelists are done like this, and must be added to "all" after each
diff --git a/libexec/spamd-setup/Makefile b/libexec/spamd-setup/Makefile
index c7145096178..7ebfb97332d 100644
--- a/libexec/spamd-setup/Makefile
+++ b/libexec/spamd-setup/Makefile
@@ -1,9 +1,12 @@
-#	$OpenBSD: Makefile,v 1.2 2003/04/28 01:34:44 deraadt Exp $
+#	$OpenBSD: Makefile,v 1.3 2004/01/21 08:07:41 deraadt Exp $
 
 PROG=	spamd-setup
 SRCS=	spamd-setup.c
 MAN=	spamd-setup.8
 
+LDADD=	-lz
+DPADD=	${LIBZ}
+
 CFLAGS+= -Wall -Wstrict-prototypes -ansi
 
 .include <bsd.prog.mk>
diff --git a/libexec/spamd-setup/spamd-setup.8 b/libexec/spamd-setup/spamd-setup.8
index 4f7a37266bd..7e34ce90371 100644
--- a/libexec/spamd-setup/spamd-setup.8
+++ b/libexec/spamd-setup/spamd-setup.8
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: spamd-setup.8,v 1.6 2003/06/02 20:20:35 jason Exp $
+.\"	$OpenBSD: spamd-setup.8,v 1.7 2004/01/21 08:07:41 deraadt Exp $
 .\"
 .\" Copyright (c) 2003 Jason L. Wright (jason@thought.net)
 .\" All rights reserved.
@@ -31,6 +31,7 @@
 .Nd parse and load file of spammer addresses
 .Sh SYNOPSIS
 .Nm spamd-setup
+.Op Fl dn
 .Op Ar file ...
 .Sh DESCRIPTION
 The
@@ -51,6 +52,15 @@ to the
 .Xr spamd 8
 daemon.
 .Pp
+The options are as follows:
+.Bl -tag -width Ds
+.It Fl d
+Debug mode reports a few pieces of information.
+.It Fl n
+Dry-run mode.  No data is shipped to
+.Xr pf 4 .
+.El
+.Pp
 Blacklists and whitelists are specified in the configuration file
 .Pa /etc/spamd.conf
 and are processed in the order specified in the
diff --git a/libexec/spamd-setup/spamd-setup.c b/libexec/spamd-setup/spamd-setup.c
index 33b60a80dab..924f869625e 100644
--- a/libexec/spamd-setup/spamd-setup.c
+++ b/libexec/spamd-setup/spamd-setup.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: spamd-setup.c,v 1.15 2004/01/21 02:49:34 deraadt Exp $ */
+/*	$OpenBSD: spamd-setup.c,v 1.16 2004/01/21 08:07:41 deraadt Exp $ */
 /*
  * Copyright (c) 2003 Bob Beck.  All rights reserved.
  *
@@ -37,6 +37,7 @@
 #include <netinet/ip_ipsp.h>
 #include <netdb.h>
 #include <machine/endian.h>
+#include <zlib.h>
 
 #define PATH_FTP		"/usr/bin/ftp"
 #define PATH_PFCTL		"/sbin/pfctl"
@@ -60,6 +61,7 @@ struct blacklist {
 	struct bl *bl;
 	size_t blc, bls;
 	u_int8_t black;
+	int count;
 };
 
 u_int32_t	imask(u_int8_t b);
@@ -76,7 +78,7 @@ int		fetch(char *url);
 int		open_file(char *method, char *file);
 char		*fix_quoted_colons(char *buf);
 void		do_message(FILE *sdc, char *msg);
-struct bl	*add_blacklist(struct bl *bl, int *blc, int *bls, int fd,
+struct bl	*add_blacklist(struct bl *bl, int *blc, int *bls, gzFile gzf,
 		    int white);
 int		cmpbl(const void *a, const void *b);
 struct cidr	**collapse_blacklist(struct bl *bl, int blc);
@@ -86,6 +88,9 @@ int		configure_pf(struct cidr **blacklists);
 int		getlist(char ** db_array, char *name, struct blacklist *blist,
 		    struct blacklist *blistnew);
 
+int		debug;
+int		dryrun;
+
 u_int32_t
 imask(u_int8_t b)
 {
@@ -281,6 +286,9 @@ fetch(char *url)
 {
 	char *argv[6]= {"ftp", "-V", "-o", "-", url, NULL};
 
+	if (debug)
+		fprintf(stderr, "Getting %s\n", url);
+
 	return open_child(PATH_FTP, argv);
 }
 
@@ -456,13 +464,13 @@ do_message(FILE *sdc, char *msg)
 
 /* retrieve a list from fd. add to blacklist bl */
 struct bl *
-add_blacklist(struct bl *bl, int *blc, int *bls, int fd, int white)
+add_blacklist(struct bl *bl, int *blc, int *bls, gzFile gzf, int white)
 {
 	int i, n, start, bu = 0, bs = 0, serrno = 0;
 	char *buf = NULL;
 
 	for (;;) {
-		/* read in fd, then parse */
+		/* read in gzf, then parse */
 		if (bu == bs) {
 			char *tmp;
 
@@ -478,7 +486,7 @@ add_blacklist(struct bl *bl, int *blc, int *bls, int fd, int white)
 			buf = tmp;
 		}
 
-		n = read(fd, buf + bu, bs - bu);
+		n = gzread(gzf, buf + bu, bs - bu);
 		if (n == 0)
 			goto parse;
 		else if (n == -1) {
@@ -504,7 +512,7 @@ add_blacklist(struct bl *bl, int *blc, int *bls, int fd, int white)
 		}
 		if (buf[i] == '\n') {
 			buf[i] = '\0';
-			if (parse_netblock (buf + start,
+			if (parse_netblock(buf + start,
 			    bl + *blc, bl + *blc + 1, white))
 				*blc+=2;
 			start = i+1;
@@ -669,6 +677,7 @@ getlist(char ** db_array, char *name, struct blacklist *blist,
 	char *buf, *method, *file, *message;
 	int blc, bls, fd, black = 0;
 	struct bl *bl = NULL;
+	gzFile gzf;
 
 	if (cgetent(&buf, db_array, name) != 0)
 		err(1, "Can't find \"%s\" in spamd config", name);
@@ -707,19 +716,23 @@ getlist(char ** db_array, char *name, struct blacklist *blist,
 
 	switch (cgetstr(buf, "file", &file)) {
 	case -1:
-		errx(1, "No file given for %slist %s", black?"black":"white",
-		    name);
+		errx(1, "No file given for %slist %s",
+		    black ? "black" : "white", name);
 	case -2:
 		errx(1, "malloc failed");
 	default:
 		fd = open_file(method, file);
 		if (fd == -1)
 			err(1, "Can't open %s by %s method",
-			    file, method ? method:"file");
+			    file, method ? method : "file");
 		free(method);
 		free(file);
+		gzf = gzdopen(fd, "r");
+		if (gzf == NULL)
+			errx(1, "gzdopen");
 	}
-	bl = add_blacklist(bl, &blc, &bls, fd, !black);
+	bl = add_blacklist(bl, &blc, &bls, gzf, !black);
+	gzclose(gzf);
 	if (bl == NULL) {
 		warn("Could not add %slist %s", black ? "black" : "white",
 		    name);
@@ -738,6 +751,9 @@ getlist(char ** db_array, char *name, struct blacklist *blist,
 		blist->blc = blc;
 		blist->bls = bls;
 	}
+	if (debug)
+		fprintf(stderr, "%slist %s %d entries\n",
+		    black ? "black" : "white", name, blc / 2);
 	return(black);
 }
 
@@ -748,10 +764,23 @@ main(int argc, char *argv[])
 	char **db_array, *buf, *name;
 	struct blacklist *blists;
 	struct servent *ent;
-	int i;
+	int i, ch;
+
+	while ((ch = getopt(argc, argv, "nd")) != -1) {
+		switch (ch) {
+		case 'n':
+			dryrun = 1;
+			break;
+		case 'd':
+			debug = 1;
+			break;
+		default:
+			break;
+		}
+	}
 
 	if ((ent = getservbyname("spamd-cfg", "tcp")) == NULL)
-		errx(1, "Can't find service \"spamd-cfg\" in /etc/services");
+		errx(1, "cannot find service \"spamd-cfg\" in /etc/services");
 	ent->s_port = ntohs(ent->s_port);
 
 	dbs = argc + 2;
@@ -803,6 +832,9 @@ main(int argc, char *argv[])
 			   blists[i].blc);
 			if (cidrs == NULL)
 				errx(1, "malloc failed");
+			if (dryrun)
+				continue;
+
 			if (configure_spamd(ent->s_port, blists[i].name,
 			    blists[i].message, cidrs) == -1)
 				err(1, "Can't connect to spamd on port %d",