Three changes to fix problems with xterm in VT340 mode, reported by

Thomas Sattler.

1) Do not include the DECSLRM or DECFRA features for xterm; they will be
   added instead if secondary DA responds as VT420 (this happens
   already).

2) Set or reset the individual flags after terminal-overrides is
   applied, so the user can properly disable them.

3) Add a capability for DECFRA ("Rect").

4 files changed, 84 insertions, 40 deletions

diff --git a/usr.bin/tmux/tmux.1 b/usr.bin/tmux/tmux.1
index dda19bd2e4a..a89476f2b8a 100644
--- a/usr.bin/tmux/tmux.1
+++ b/usr.bin/tmux/tmux.1
@@ -1,4 +1,4 @@
-.\" $OpenBSD: tmux.1,v 1.836 2021/06/10 07:24:10 nicm Exp $
+.\" $OpenBSD: tmux.1,v 1.837 2021/06/10 07:28:45 nicm Exp $
 .\"
 .\" Copyright (c) 2007 Nicholas Marriott <nicholas.marriott@gmail.com>
 .\"
@@ -6049,6 +6049,10 @@ Disable and enable focus reporting.
 These are set automatically if the
 .Em XT
 capability is present.
+.It Em \&Rect
+Tell
+.Nm
+that the terminal supports rectangle operations.
 .It Em \&Smol
 Enable the overline attribute.
 .It Em \&Smulx
diff --git a/usr.bin/tmux/tmux.h b/usr.bin/tmux/tmux.h
index 42a4cef21e7..82c87715030 100644
--- a/usr.bin/tmux/tmux.h
+++ b/usr.bin/tmux/tmux.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: tmux.h,v 1.1103 2021/04/12 09:36:12 nicm Exp $ */
+/* $OpenBSD: tmux.h,v 1.1104 2021/06/10 07:28:45 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicholas.marriott@gmail.com>
@@ -451,6 +451,7 @@ enum tty_code_code {
 	TTYC_MS,
 	TTYC_OL,
 	TTYC_OP,
+	TTYC_RECT,
 	TTYC_REV,
 	TTYC_RGB,
 	TTYC_RI,
diff --git a/usr.bin/tmux/tty-features.c b/usr.bin/tmux/tty-features.c
index 8e1a0d25b9e..3a055702b3d 100644
--- a/usr.bin/tmux/tty-features.c
+++ b/usr.bin/tmux/tty-features.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tty-features.c,v 1.18 2020/10/05 09:53:01 nicm Exp $ */
+/* $OpenBSD: tty-features.c,v 1.19 2021/06/10 07:28:45 nicm Exp $ */
 
 /*
  * Copyright (c) 2020 Nicholas Marriott <nicholas.marriott@gmail.com>
@@ -218,9 +218,13 @@ static const struct tty_feature tty_feature_margins = {
 };
 
 /* Terminal supports DECFRA rectangle fill. */
+static const char *tty_feature_rectfill_capabilities[] = {
+	"Rect",
+	NULL
+};
 static const struct tty_feature tty_feature_rectfill = {
 	"rectfill",
-	NULL,
+	tty_feature_rectfill_capabilities,
 	TERM_DECFRA
 };
 
@@ -351,8 +355,13 @@ tty_default_features(int *feat, const char *name, u_int version)
 			      ",cstyle,extkeys,margins,sync"
 		},
 		{ .name = "XTerm",
+		  /*
+		   * xterm also supports DECSLRM and DECFRA, but they can be
+		   * disabled so not set it here - they will be added if
+		   * secondary DA shows VT420.
+		   */
 		  .features = TTY_FEATURES_BASE_MODERN_XTERM
-			      ",ccolour,cstyle,extkeys,focus,margins,rectfill"
+			      ",ccolour,cstyle,extkeys,focus"
 		}
 	};
 	u_int	i;
diff --git a/usr.bin/tmux/tty-term.c b/usr.bin/tmux/tty-term.c
index 6ed648a516d..8c44fa92b8f 100644
--- a/usr.bin/tmux/tty-term.c
+++ b/usr.bin/tmux/tty-term.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tty-term.c,v 1.88 2021/03/01 10:50:14 nicm Exp $ */
+/* $OpenBSD: tty-term.c,v 1.89 2021/06/10 07:28:45 nicm Exp $ */
 
 /*
  * Copyright (c) 2008 Nicholas Marriott <nicholas.marriott@gmail.com>
@@ -248,6 +248,7 @@ static const struct tty_term_code_entry tty_term_codes[] = {
 	[TTYC_MS] = { TTYCODE_STRING, "Ms" },
 	[TTYC_OL] = { TTYCODE_STRING, "ol" },
 	[TTYC_OP] = { TTYCODE_STRING, "op" },
+	[TTYC_RECT] = { TTYCODE_STRING, "Rect" },
 	[TTYC_REV] = { TTYCODE_STRING, "rev" },
 	[TTYC_RGB] = { TTYCODE_FLAG, "RGB" },
 	[TTYC_RIN] = { TTYCODE_STRING, "rin" },
@@ -431,10 +432,11 @@ tty_term_apply_overrides(struct tty_term *term)
 	struct options_entry		*o;
 	struct options_array_item	*a;
 	union options_value		*ov;
-	const char			*s;
+	const char			*s, *acs;
 	size_t				 offset;
 	char				*first;
 
+	/* Update capabilities from the option. */
 	o = options_get_only(global_options, "terminal-overrides");
 	a = options_array_first(o);
 	while (a != NULL) {
@@ -447,6 +449,64 @@ tty_term_apply_overrides(struct tty_term *term)
 			tty_term_apply(term, s + offset, 0);
 		a = options_array_next(a);
 	}
+
+	/* Update the RGB flag if the terminal has RGB colours. */
+	if (tty_term_has(term, TTYC_SETRGBF) &&
+	    tty_term_has(term, TTYC_SETRGBB))
+		term->flags |= TERM_RGBCOLOURS;
+	else
+		term->flags &= ~TERM_RGBCOLOURS;
+	log_debug("RGBCOLOURS flag is %d", !!(term->flags & TERM_RGBCOLOURS));
+
+	/*
+	 * Set or clear the DECSLRM flag if the terminal has the margin
+	 * capabilities.
+	 */
+	if (tty_term_has(term, TTYC_CMG) && tty_term_has(term, TTYC_CLMG))
+		term->flags |= TERM_DECSLRM;
+	else
+		term->flags &= ~TERM_DECSLRM;
+	log_debug("DECSLRM flag is %d", !!(term->flags & TERM_DECSLRM));
+
+	/*
+	 * Set or clear the DECFRA flag if the terminal has the rectangle
+	 * capability.
+	 */
+	if (tty_term_has(term, TTYC_RECT))
+		term->flags |= TERM_DECFRA;
+	else
+		term->flags &= ~TERM_DECFRA;
+	log_debug("DECFRA flag is %d", !!(term->flags & TERM_DECFRA));
+
+	/*
+	 * Terminals without am (auto right margin) wrap at at $COLUMNS - 1
+	 * rather than $COLUMNS (the cursor can never be beyond $COLUMNS - 1).
+	 *
+	 * Terminals without xenl (eat newline glitch) ignore a newline beyond
+	 * the right edge of the terminal, but tmux doesn't care about this -
+	 * it always uses absolute only moves the cursor with a newline when
+	 * also sending a linefeed.
+	 *
+	 * This is irritating, most notably because it is painful to write to
+	 * the very bottom-right of the screen without scrolling.
+	 *
+	 * Flag the terminal here and apply some workarounds in other places to
+	 * do the best possible.
+	 */
+	if (!tty_term_flag(term, TTYC_AM))
+		term->flags |= TERM_NOAM;
+	else
+		term->flags &= ~TERM_NOAM;
+	log_debug("NOAM flag is %d", !!(term->flags & TERM_NOAM));
+
+	/* Generate ACS table. If none is present, use nearest ASCII. */
+	memset(term->acs, 0, sizeof term->acs);
+	if (tty_term_has(term, TTYC_ACSC))
+		acs = tty_term_string(term, TTYC_ACSC);
+	else
+		acs = "a#j+k+l+m+n+o-p-q-r-s-t+u+v+w+x|y<z>~.";
+	for (; acs[0] != '\0' && acs[1] != '\0'; acs += 2)
+		term->acs[(u_char) acs[0]][0] = acs[1];
 }
 
 struct tty_term *
@@ -460,7 +520,7 @@ tty_term_create(struct tty *tty, char *name, char **caps, u_int ncaps,
 	struct options_array_item		*a;
 	union options_value			*ov;
 	u_int					 i, j;
-	const char				*s, *acs, *value;
+	const char				*s, *value;
 	size_t					 offset, namelen;
 	char					*first;
 
@@ -557,40 +617,10 @@ tty_term_create(struct tty *tty, char *name, char **caps, u_int ncaps,
 	    (!tty_term_has(term, TTYC_SETRGBF) ||
 	    !tty_term_has(term, TTYC_SETRGBB)))
 		tty_add_features(feat, "RGB", ",");
-	if (tty_term_has(term, TTYC_SETRGBF) &&
-	    tty_term_has(term, TTYC_SETRGBB))
-		term->flags |= TERM_RGBCOLOURS;
 
 	/* Apply the features and overrides again. */
-	tty_apply_features(term, *feat);
-	tty_term_apply_overrides(term);
-
-	/*
-	 * Terminals without am (auto right margin) wrap at at $COLUMNS - 1
-	 * rather than $COLUMNS (the cursor can never be beyond $COLUMNS - 1).
-	 *
-	 * Terminals without xenl (eat newline glitch) ignore a newline beyond
-	 * the right edge of the terminal, but tmux doesn't care about this -
-	 * it always uses absolute only moves the cursor with a newline when
-	 * also sending a linefeed.
-	 *
-	 * This is irritating, most notably because it is painful to write to
-	 * the very bottom-right of the screen without scrolling.
-	 *
-	 * Flag the terminal here and apply some workarounds in other places to
-	 * do the best possible.
-	 */
-	if (!tty_term_flag(term, TTYC_AM))
-		term->flags |= TERM_NOAM;
-
-	/* Generate ACS table. If none is present, use nearest ASCII. */
-	memset(term->acs, 0, sizeof term->acs);
-	if (tty_term_has(term, TTYC_ACSC))
-		acs = tty_term_string(term, TTYC_ACSC);
-	else
-		acs = "a#j+k+l+m+n+o-p-q-r-s-t+u+v+w+x|y<z>~.";
-	for (; acs[0] != '\0' && acs[1] != '\0'; acs += 2)
-		term->acs[(u_char) acs[0]][0] = acs[1];
+	if (tty_apply_features(term, *feat))
+		tty_term_apply_overrides(term);
 
 	/* Log the capabilities. */
 	for (i = 0; i < tty_term_ncodes(); i++)