summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlexander Bluhm <bluhm@cvs.openbsd.org>2015-10-09 17:07:07 +0000
committerAlexander Bluhm <bluhm@cvs.openbsd.org>2015-10-09 17:07:07 +0000
commita9ab6eeaaba00c2f5408dac88e8abcd3fd33abd3 (patch)
tree0c6f53481fea8a946d876b35dd1d98ad5c7d0a65
parentef8f386fa7076cd0369144fdaca6dc0e905e6c93 (diff)
Add tests for syslogd TLS accept and receive encrypted messages.
-rw-r--r--regress/usr.sbin/syslogd/Client.pm29
-rw-r--r--regress/usr.sbin/syslogd/Makefile13
-rw-r--r--regress/usr.sbin/syslogd/args-client-tcp-close.pl51
-rw-r--r--regress/usr.sbin/syslogd/args-client-tcp-error.pl53
-rw-r--r--regress/usr.sbin/syslogd/args-client-tcp.pl4
-rw-r--r--regress/usr.sbin/syslogd/args-client-tls-close.pl51
-rw-r--r--regress/usr.sbin/syslogd/args-client-tls-error.pl54
-rw-r--r--regress/usr.sbin/syslogd/args-client-tls-tcp.pl53
-rw-r--r--regress/usr.sbin/syslogd/args-client-tls.pl47
-rw-r--r--regress/usr.sbin/syslogd/args-client-tls4.pl47
-rw-r--r--regress/usr.sbin/syslogd/args-client-tls6.pl47
-rw-r--r--regress/usr.sbin/syslogd/args-tls-cafile-default.pl5
-rw-r--r--regress/usr.sbin/syslogd/funcs.pl4
13 files changed, 449 insertions, 9 deletions
diff --git a/regress/usr.sbin/syslogd/Client.pm b/regress/usr.sbin/syslogd/Client.pm
index af2904d7f70..e7656827ee3 100644
--- a/regress/usr.sbin/syslogd/Client.pm
+++ b/regress/usr.sbin/syslogd/Client.pm
@@ -1,4 +1,4 @@
-# $OpenBSD: Client.pm,v 1.4 2015/09/09 08:48:46 bluhm Exp $
+# $OpenBSD: Client.pm,v 1.5 2015/10/09 17:07:06 bluhm Exp $
# Copyright (c) 2010-2014 Alexander Bluhm <bluhm@openbsd.org>
#
@@ -20,7 +20,11 @@ use warnings;
package Client;
use parent 'Proc';
use Carp;
+use Socket;
+use Socket6;
+use IO::Socket;
use IO::Socket::INET6;
+use IO::Socket::SSL;
use Sys::Syslog qw(:standard :extended :macros);
sub new {
@@ -48,16 +52,33 @@ sub child {
$cs->setsockopt(SOL_SOCKET, SO_SNDBUF, 10000)
or die ref($self), " setsockopt failed: $!";
} else {
- $cs = IO::Socket::INET6->new(
- Proto => $self->{connectproto},
+ $SSL_ERROR = "";
+ my $iosocket = $self->{connectproto} eq "tls" ?
+ "IO::Socket::SSL" : "IO::Socket::INET6";
+ my $proto = $self->{connectproto};
+ $proto = "tcp" if $proto eq "tls";
+ $cs = $iosocket->new(
+ Proto => $proto,
Domain => $self->{connectdomain},
PeerAddr => $self->{connectaddr},
PeerPort => $self->{connectport},
- ) or die ref($self), " socket connect failed: $!";
+ SSL_verify_mode => SSL_VERIFY_NONE,
+ $self->{sslversion} ?
+ (SSL_version => $self->{sslversion}) : (),
+ $self->{sslciphers} ?
+ (SSL_cipher_list => $self->{sslciphers}) : (),
+ ) or die ref($self), " $iosocket socket connect ".
+ "failed: $!,$SSL_ERROR";
print STDERR "connect sock: ",$cs->sockhost()," ",
$cs->sockport(),"\n";
print STDERR "connect peer: ",$cs->peerhost()," ",
$cs->peerport(),"\n";
+ if ($self->{connectproto} eq "tls") {
+ print STDERR "ssl version: ",
+ $cs->get_sslversion(),"\n";
+ print STDERR "ssl cipher: ",
+ $cs->get_cipher(),"\n";
+ }
}
*STDIN = *STDOUT = $self->{cs} = $cs;
diff --git a/regress/usr.sbin/syslogd/Makefile b/regress/usr.sbin/syslogd/Makefile
index a60a7ea7d76..8f538a979d2 100644
--- a/regress/usr.sbin/syslogd/Makefile
+++ b/regress/usr.sbin/syslogd/Makefile
@@ -1,4 +1,4 @@
-# $OpenBSD: Makefile,v 1.11 2015/09/11 22:01:06 bluhm Exp $
+# $OpenBSD: Makefile,v 1.12 2015/10/09 17:07:06 bluhm Exp $
# The following ports must be installed for the regression tests:
# p5-IO-Socket-INET6 object interface for AF_INET and AF_INET6 domain sockets
@@ -71,6 +71,15 @@ run-regress-$a: $a
# create certificates for TLS
+127.0.0.1.crt: ca.crt
+ openssl req -batch -new -subj /L=OpenBSD/O=syslogd-regress/OU=syslogd/CN=127.0.0.1/ -nodes -newkey rsa -keyout 127.0.0.1.key -x509 -out $@
+ ${SUDO} cp 127.0.0.1.crt /etc/ssl/
+ ${SUDO} cp 127.0.0.1.key /etc/ssl/private/
+ ${SUDO} cp 127.0.0.1.crt /etc/ssl/localhost.crt
+ ${SUDO} cp 127.0.0.1.key /etc/ssl/private/localhost.key
+ ${SUDO} cp 127.0.0.1.crt /etc/ssl/::1.crt
+ ${SUDO} cp 127.0.0.1.key /etc/ssl/private/::1.key
+
ca.crt fake-ca.crt:
openssl req -batch -new -subj /L=OpenBSD/O=syslogd-regress/OU=ca/CN=root/ -nodes -newkey rsa -keyout ${@:R}.key -x509 -out $@
@@ -86,7 +95,7 @@ empty:
toobig:
dd if=/dev/zero of=$@ bs=1 count=1 seek=50M
-${REGRESS_TARGETS:M*tls*}: server.crt
+${REGRESS_TARGETS:M*tls*}: server.crt 127.0.0.1.crt
${REGRESS_TARGETS:M*empty*}: empty
${REGRESS_TARGETS:M*toobig*}: toobig
${REGRESS_TARGETS:M*fake*}: fake-ca.crt
diff --git a/regress/usr.sbin/syslogd/args-client-tcp-close.pl b/regress/usr.sbin/syslogd/args-client-tcp-close.pl
new file mode 100644
index 00000000000..174e658cbcb
--- /dev/null
+++ b/regress/usr.sbin/syslogd/args-client-tcp-close.pl
@@ -0,0 +1,51 @@
+# The syslogd listens on 127.0.0.1 TLS socket.
+# The client connects and closes the connection to syslogd.
+# The syslogd writes the error into a file and through a pipe.
+# Find the message in file, syslogd log.
+# Check that syslogd writes a log message about the client close.
+
+use strict;
+use warnings;
+use Socket;
+
+our %args = (
+ client => {
+ connect => { domain => AF_INET, proto => "tcp", addr => "127.0.0.1",
+ port => 514 },
+ func => sub {
+ my $self = shift;
+ shutdown(\*STDOUT, 1)
+ or die "shutdown write failed: $!";
+ ${$self->{syslogd}}->loggrep("tcp logger .* connection close", 5)
+ or die "no connection close in syslogd.log";
+ },
+ loggrep => {
+ qr/connect sock: 127.0.0.1 \d+/ => 1,
+ },
+ },
+ syslogd => {
+ options => ["-T", "127.0.0.1:514"],
+ loggrep => {
+ qr/syslogd: tcp logger .* accepted/ => 1,
+ qr/syslogd: tcp logger .* connection close/ => 1,
+ }
+ },
+ server => {
+ func => sub {
+ my $self = shift;
+ ${$self->{syslogd}}->loggrep("tcp logger .* connection close", 5)
+ or die "no connection close in syslogd.log";
+ },
+ loggrep => {},
+ },
+ pipe => {
+ loggrep => {},
+ },
+ file => {
+ loggrep => {
+ qr/syslogd: tcp logger .* connection close/ => 1,
+ },
+ },
+);
+
+1;
diff --git a/regress/usr.sbin/syslogd/args-client-tcp-error.pl b/regress/usr.sbin/syslogd/args-client-tcp-error.pl
new file mode 100644
index 00000000000..dd7c68df8bf
--- /dev/null
+++ b/regress/usr.sbin/syslogd/args-client-tcp-error.pl
@@ -0,0 +1,53 @@
+# The syslogd listens on 127.0.0.1 TCP socket.
+# The client connects and aborts the connection to syslogd.
+# The syslogd writes the error into a file and through a pipe.
+# Find the message in file, syslogd log.
+# Check that syslogd writes a log message about the client error.
+
+use strict;
+use warnings;
+use Socket;
+use Errno ':POSIX';
+
+my @errors = (ECONNRESET);
+my $errors = "(". join("|", map { $! = $_ } @errors). ")";
+
+our %args = (
+ client => {
+ connect => { domain => AF_INET, proto => "tcp", addr => "127.0.0.1",
+ port => 514 },
+ func => sub {
+ my $self = shift;
+ setsockopt(STDOUT, SOL_SOCKET, SO_LINGER, pack('ii', 1, 0))
+ or die "set socket linger failed: $!";
+ },
+ loggrep => {
+ qr/connect sock: 127.0.0.1 \d+/ => 1,
+ },
+ },
+ syslogd => {
+ options => ["-T", "127.0.0.1:514"],
+ loggrep => {
+ qr/syslogd: tcp logger .* accept/ => 1,
+ qr/syslogd: tcp logger .* connection error/ => 1,
+ },
+ },
+ server => {
+ func => sub {
+ my $self = shift;
+ ${$self->{syslogd}}->loggrep("tcp logger .* connection error", 5)
+ or die "no connection error in syslogd.log";
+ },
+ loggrep => {},
+ },
+ pipe => {
+ loggrep => {},
+ },
+ file => {
+ loggrep => {
+ qr/syslogd: tcp logger .* connection error: $errors/ => 1,
+ },
+ },
+);
+
+1;
diff --git a/regress/usr.sbin/syslogd/args-client-tcp.pl b/regress/usr.sbin/syslogd/args-client-tcp.pl
index 5e4ea942f88..b047eecc264 100644
--- a/regress/usr.sbin/syslogd/args-client-tcp.pl
+++ b/regress/usr.sbin/syslogd/args-client-tcp.pl
@@ -20,6 +20,10 @@ our %args = (
qr/^_syslogd .* internet/ => 3,
qr/ internet6? stream tcp \w+ (127.0.0.1|\[::1\]):514$/ => 1,
},
+ loggrep => {
+ qr/syslogd: tcp logger .* accepted/ => 1,
+ qr/syslogd: tcp logger .* connection close/ => 1,
+ },
},
file => {
loggrep => qr/ localhost syslogd-regress\[\d+\]: /. get_testgrep(),
diff --git a/regress/usr.sbin/syslogd/args-client-tls-close.pl b/regress/usr.sbin/syslogd/args-client-tls-close.pl
new file mode 100644
index 00000000000..ac9a1b13313
--- /dev/null
+++ b/regress/usr.sbin/syslogd/args-client-tls-close.pl
@@ -0,0 +1,51 @@
+# The syslogd listens on 127.0.0.1 TLS socket.
+# The client connects and closes the connection to syslogd.
+# The syslogd writes the error into a file and through a pipe.
+# Find the error message in file, syslogd log.
+# Check that syslogd writes a log message about the client close.
+
+use strict;
+use warnings;
+use Socket;
+
+our %args = (
+ client => {
+ connect => { domain => AF_INET, proto => "tls", addr => "127.0.0.1",
+ port => 6514 },
+ func => sub {
+ my $self = shift;
+ shutdown(\*STDOUT, 1)
+ or die "shutdown write failed: $!";
+ ${$self->{syslogd}}->loggrep("tls logger .* connection close", 5)
+ or die "no connection close in syslogd.log";
+ },
+ loggrep => {
+ qr/connect sock: 127.0.0.1 \d+/ => 1,
+ },
+ },
+ syslogd => {
+ options => ["-S", "127.0.0.1:6514"],
+ loggrep => {
+ qr/syslogd: tls logger .* accepted/ => 1,
+ qr/syslogd: tls logger .* connection close/ => 1,
+ },
+ },
+ server => {
+ func => sub {
+ my $self = shift;
+ ${$self->{syslogd}}->loggrep("tls logger .* connection close", 5)
+ or die "no connection close in syslogd.log";
+ },
+ loggrep => {},
+ },
+ pipe => {
+ loggrep => {},
+ },
+ file => {
+ loggrep => {
+ qr/syslogd: tls logger .* connection close/ => 1,
+ },
+ },
+);
+
+1;
diff --git a/regress/usr.sbin/syslogd/args-client-tls-error.pl b/regress/usr.sbin/syslogd/args-client-tls-error.pl
new file mode 100644
index 00000000000..12b67a0f955
--- /dev/null
+++ b/regress/usr.sbin/syslogd/args-client-tls-error.pl
@@ -0,0 +1,54 @@
+# The syslogd listens on 127.0.0.1 TLS socket.
+# The client connects and aborts the connection to syslogd.
+# The syslogd writes the error into a file and through a pipe.
+# Find the error message in file, syslogd log.
+# Check that syslogd writes a log message about the client error.
+
+use strict;
+use warnings;
+use Socket;
+use Errno ':POSIX';
+
+my @errors = (ECONNRESET);
+my $errors = "(". join("|", map { $! = $_ } @errors). ")";
+
+our %args = (
+ client => {
+ connect => { domain => AF_INET, proto => "tls", addr => "127.0.0.1",
+ port => 6514 },
+ func => sub {
+ my $self = shift;
+ setsockopt(STDOUT, SOL_SOCKET, SO_LINGER, pack('ii', 1, 0))
+ or die "set socket linger failed: $!";
+ },
+ loggrep => {
+ qr/connect sock: 127.0.0.1 \d+/ => 1,
+ },
+ },
+ syslogd => {
+ options => ["-S", "127.0.0.1:6514"],
+ loggrep => {
+ qr/syslogd: tls logger .* accept/ => 1,
+ qr/syslogd: tls logger .* connection error/ => 1,
+ },
+ },
+ server => {
+ func => sub {
+ my $self = shift;
+ ${$self->{syslogd}}->loggrep("tls logger .* connection error", 5)
+ or die "no connection error in syslogd.log";
+ },
+ loggrep => {},
+ },
+ pipe => {
+ loggrep => {},
+ },
+ file => {
+ loggrep => {
+ qr/syslogd: tls logger .* connection error: read failed: $errors/
+ => 1,
+ },
+ },
+);
+
+1;
diff --git a/regress/usr.sbin/syslogd/args-client-tls-tcp.pl b/regress/usr.sbin/syslogd/args-client-tls-tcp.pl
new file mode 100644
index 00000000000..5613fc2b129
--- /dev/null
+++ b/regress/usr.sbin/syslogd/args-client-tls-tcp.pl
@@ -0,0 +1,53 @@
+# The syslogd listens on 127.0.0.1 TLS socket.
+# The TCP client writes cleartext into the TLS connection to syslogd.
+# The client connects and closes the connection to syslogd.
+# The syslogd writes the error into a file and through a pipe.
+# Find the error message in file, syslogd log.
+# Check that syslogd writes a log message about the SSL connect error.
+
+use strict;
+use warnings;
+use Socket;
+
+our %args = (
+ client => {
+ connect => { domain => AF_INET, proto => "tcp", addr => "127.0.0.1",
+ port => 6514 },
+ func => sub {
+ my $self = shift;
+ print "Writing cleartext into a TLS connection is a bad idea\n";
+ ${$self->{syslogd}}->loggrep("tls logger .* connection error", 5)
+ or die "no connection error in syslogd.log";
+ },
+ loggrep => {
+ qr/connect sock: 127.0.0.1 \d+/ => 1,
+ },
+ },
+ syslogd => {
+ options => ["-S", "127.0.0.1:6514"],
+ loggrep => {
+ qr/syslogd: tls logger .* accepted/ => 1,
+ qr/syslogd: tls logger .* connection error/ => 1,
+ },
+ },
+ server => {
+ func => sub {
+ my $self = shift;
+ ${$self->{syslogd}}->loggrep("tls logger .* connection error", 5)
+ or die "no connection error in syslogd.log";
+ },
+ loggrep => {},
+ },
+ pipe => {
+ loggrep => {},
+ },
+ file => {
+ loggrep => {
+ qr/syslogd: tls logger .* connection error: /.
+ qr/handshake failed: error:.*/.
+ qr/SSL routines:SSL23_GET_CLIENT_HELLO:unknown protocol/ => 1,
+ },
+ },
+);
+
+1;
diff --git a/regress/usr.sbin/syslogd/args-client-tls.pl b/regress/usr.sbin/syslogd/args-client-tls.pl
new file mode 100644
index 00000000000..60f47f2687e
--- /dev/null
+++ b/regress/usr.sbin/syslogd/args-client-tls.pl
@@ -0,0 +1,47 @@
+# The syslogd listens on localhost TLS socket.
+# The client writes a message into a localhost TLS socket.
+# The syslogd writes it into a file and through a pipe.
+# The syslogd passes it via UDP to the loghost.
+# The server receives the message on its UDP socket.
+# Find the message in client, file, pipe, syslogd, server log.
+# Check that the file log contains the hostname and message.
+
+use strict;
+use warnings;
+use Socket;
+
+our %args = (
+ client => {
+ connect => { domain => AF_UNSPEC, proto => "tls", addr => "localhost",
+ port => 6514 },
+ loggrep => {
+ qr/connect sock: (127.0.0.1|::1) \d+/ => 1,
+ get_testgrep() => 1,
+ },
+ },
+ syslogd => {
+ options => ["-S", "localhost"],
+ fstat => {
+ qr/^root .* internet/ => 0,
+ qr/^_syslogd .* internet/ => 3,
+ qr/ internet6? stream tcp \w+ (127.0.0.1|\[::1\]):6514$/ => 1,
+ },
+ ktrace => {
+ qr{NAMI "/etc/ssl/private/localhost.key"} => 1,
+ qr{NAMI "/etc/ssl/localhost.crt"} => 1,
+ },
+ loggrep => {
+ qr{Keyfile /etc/ssl/private/localhost.key} => 1,
+ qr{Certfile /etc/ssl/localhost.crt} => 1,
+ qr/syslogd: tls logger .* accepted/ => 1,
+ qr/syslogd: tls logger .* connection close/ => 1,
+ },
+ },
+ file => {
+ loggrep => {
+ qr/ localhost /. get_testgrep() => 1,
+ },
+ },
+);
+
+1;
diff --git a/regress/usr.sbin/syslogd/args-client-tls4.pl b/regress/usr.sbin/syslogd/args-client-tls4.pl
new file mode 100644
index 00000000000..84b104e9278
--- /dev/null
+++ b/regress/usr.sbin/syslogd/args-client-tls4.pl
@@ -0,0 +1,47 @@
+# The syslogd listens on 127.0.0.1 TLS socket.
+# The client writes a message into a 127.0.0.1 TLS socket.
+# The syslogd writes it into a file and through a pipe.
+# The syslogd passes it via UDP to the loghost.
+# The server receives the message on its UDP socket.
+# Find the message in client, file, pipe, syslogd, server log.
+# Check that the file log contains the hostname and message.
+
+use strict;
+use warnings;
+use Socket;
+
+our %args = (
+ client => {
+ connect => { domain => AF_INET, proto => "tls", addr => "127.0.0.1",
+ port => 6514 },
+ loggrep => {
+ qr/connect sock: 127.0.0.1 \d+/ => 1,
+ get_testgrep() => 1,
+ },
+ },
+ syslogd => {
+ options => ["-S", "127.0.0.1:6514"],
+ fstat => {
+ qr/^root .* internet/ => 0,
+ qr/^_syslogd .* internet/ => 3,
+ qr/ internet stream tcp \w+ 127.0.0.1:6514$/ => 1,
+ },
+ ktrace => {
+ qr{NAMI "/etc/ssl/private/127.0.0.1:6514.key"} => 1,
+ qr{NAMI "/etc/ssl/private/127.0.0.1.key"} => 1,
+ qr{NAMI "/etc/ssl/127.0.0.1:6514.crt"} => 1,
+ qr{NAMI "/etc/ssl/127.0.0.1.crt"} => 1,
+ },
+ loggrep => {
+ qr{Keyfile /etc/ssl/private/127.0.0.1.key} => 1,
+ qr{Certfile /etc/ssl/127.0.0.1.crt} => 1,
+ },
+ },
+ file => {
+ loggrep => {
+ qr/ localhost /. get_testgrep() => 1,
+ },
+ },
+);
+
+1;
diff --git a/regress/usr.sbin/syslogd/args-client-tls6.pl b/regress/usr.sbin/syslogd/args-client-tls6.pl
new file mode 100644
index 00000000000..aa51ca031e6
--- /dev/null
+++ b/regress/usr.sbin/syslogd/args-client-tls6.pl
@@ -0,0 +1,47 @@
+# The syslogd listens on ::1 TLS socket.
+# The client writes a message into a ::1 TLS socket.
+# The syslogd writes it into a file and through a pipe.
+# The syslogd passes it via UDP to the loghost.
+# The server receives the message on its UDP socket.
+# Find the message in client, file, pipe, syslogd, server log.
+# Check that the file log contains the hostname and message.
+
+use strict;
+use warnings;
+use Socket;
+
+our %args = (
+ client => {
+ connect => { domain => AF_INET6, proto => "tls", addr => "::1",
+ port => 6514 },
+ loggrep => {
+ qr/connect sock: ::1 \d+/ => 1,
+ get_testgrep() => 1,
+ },
+ },
+ syslogd => {
+ options => ["-S", "[::1]:6514"],
+ fstat => {
+ qr/^root .* internet/ => 0,
+ qr/^_syslogd .* internet/ => 3,
+ qr/ internet6 stream tcp \w+ \[::1\]:6514$/ => 1,
+ },
+ ktrace => {
+ qr{NAMI "/etc/ssl/private/\[::1\]:6514.key"} => 1,
+ qr{NAMI "/etc/ssl/private/::1.key"} => 1,
+ qr{NAMI "/etc/ssl/\[::1\]:6514.crt"} => 1,
+ qr{NAMI "/etc/ssl/::1.crt"} => 1,
+ },
+ loggrep => {
+ qr{Keyfile /etc/ssl/private/::1.key} => 1,
+ qr{Certfile /etc/ssl/::1.crt} => 1,
+ },
+ },
+ file => {
+ loggrep => {
+ qr/ localhost /. get_testgrep() => 1,
+ },
+ },
+);
+
+1;
diff --git a/regress/usr.sbin/syslogd/args-tls-cafile-default.pl b/regress/usr.sbin/syslogd/args-tls-cafile-default.pl
index f47d8386df4..6fecd90983f 100644
--- a/regress/usr.sbin/syslogd/args-tls-cafile-default.pl
+++ b/regress/usr.sbin/syslogd/args-tls-cafile-default.pl
@@ -12,8 +12,11 @@ use Socket;
our %args = (
syslogd => {
loghost => '@tls://localhost:$connectport',
+ ktrace => {
+ qr{NAMI "/etc/ssl/cert.pem"} => 1,
+ },
loggrep => {
- qr/CAfile \/etc\/ssl\/cert.pem/ => 1,
+ qr{CAfile /etc/ssl/cert.pem} => 1,
qr/Logging to FORWTLS \@tls:\/\/localhost:\d+/ => '>=4',
qr/syslogd: loghost .* connection error: /.
qr/handshake failed: error:.*/.
diff --git a/regress/usr.sbin/syslogd/funcs.pl b/regress/usr.sbin/syslogd/funcs.pl
index a11b63166c1..2eeead911f8 100644
--- a/regress/usr.sbin/syslogd/funcs.pl
+++ b/regress/usr.sbin/syslogd/funcs.pl
@@ -1,4 +1,4 @@
-# $OpenBSD: funcs.pl,v 1.24 2015/09/09 08:48:46 bluhm Exp $
+# $OpenBSD: funcs.pl,v 1.25 2015/10/09 17:07:06 bluhm Exp $
# Copyright (c) 2010-2015 Alexander Bluhm <bluhm@openbsd.org>
#
@@ -96,7 +96,7 @@ sub write_message {
or die ref($self), " short UDP write";
} else {
print $msg;
- print "\n" if $self->{connectproto} eq "tcp";
+ print "\n" if $self->{connectproto} =~ /^(tcp|tls)$/;
}
print STDERR "<<< $msg\n";
} else {