Tidy up new UTF-8 code and make it more generic.

4 files changed, 147 insertions, 157 deletions

diff --git a/usr.bin/tmux/grid.c b/usr.bin/tmux/grid.c
index d2cc6a2efc3..15daac56629 100644
--- a/usr.bin/tmux/grid.c
+++ b/usr.bin/tmux/grid.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: grid.c,v 1.110 2020/05/25 09:32:10 nicm Exp $ */
+/* $OpenBSD: grid.c,v 1.111 2020/05/25 18:19:29 nicm Exp $ */
 
 /*
  * Copyright (c) 2008 Nicholas Marriott <nicholas.marriott@gmail.com>
@@ -114,7 +114,7 @@ grid_extended_cell(struct grid_line *gl, struct grid_cell_entry *gce,
 	gl->flags |= GRID_LINE_EXTENDED;
 
 	gee = &gl->extddata[gce->offset];
-	gee->data = utf8_map_big(&gc->data);
+	utf8_from_data(&gc->data, &gee->data);
 	gee->attr = gc->attr;
 	gee->flags = flags;
 	gee->fg = gc->fg;
@@ -496,7 +496,7 @@ grid_get_cell1(struct grid_line *gl, u_int px, struct grid_cell *gc)
 			gc->fg = gee->fg;
 			gc->bg = gee->bg;
 			gc->us = gee->us;
-			utf8_get_big(gee->data, &gc->data);
+			utf8_to_data(gee->data, &gc->data);
 		}
 		return;
 	}
@@ -570,7 +570,7 @@ grid_set_cells(struct grid *gd, u_int px, u_int py, const struct grid_cell *gc,
 		gce = &gl->celldata[px + i];
 		if (grid_need_extended_cell(gce, gc)) {
 			gee = grid_extended_cell(gl, gce, gc);
-			gee->data = utf8_set_big(s[i], 1);
+			gee->data = utf8_build_one(s[i], 1);
 		} else
 			grid_store_cell(gce, gc, s[i]);
 	}
diff --git a/usr.bin/tmux/tmux.h b/usr.bin/tmux/tmux.h
index 636cb64346f..8370e855efc 100644
--- a/usr.bin/tmux/tmux.h
+++ b/usr.bin/tmux/tmux.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: tmux.h,v 1.1054 2020/05/25 15:02:25 nicm Exp $ */
+/* $OpenBSD: tmux.h,v 1.1055 2020/05/25 18:19:29 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicholas.marriott@gmail.com>
@@ -596,10 +596,13 @@ struct msg_write_close {
 #define ALL_MOUSE_MODES (MODE_MOUSE_STANDARD|MODE_MOUSE_BUTTON|MODE_MOUSE_ALL)
 #define MOTION_MOUSE_MODES (MODE_MOUSE_BUTTON|MODE_MOUSE_ALL)
 
+/* A single UTF-8 character. */
+typedef u_int utf8_char;
+
 /*
- * A single UTF-8 character. UTF8_SIZE must be big enough to hold combining
+ * An expanded UTF-8 character. UTF8_SIZE must be big enough to hold combining
  * characters as well. It can't be more than 32 bytes without changes to how
- * big characters are stored.
+ * characters are stored.
  */
 #define UTF8_SIZE 21
 struct utf8_data {
@@ -673,7 +676,7 @@ struct grid_cell {
 
 /* Grid extended cell entry. */
 struct grid_extd_entry {
-	u_int			data;
+	utf8_char		data;
 	u_short			attr;
 	u_char			flags;
 	int			fg;
@@ -2889,9 +2892,9 @@ u_int		 session_group_attached_count(struct session_group *);
 void		 session_renumber_windows(struct session *);
 
 /* utf8.c */
-u_int		 utf8_set_big(char, u_int);
-u_int		 utf8_map_big(const struct utf8_data *);
-void		 utf8_get_big(u_int, struct utf8_data *);
+utf8_char	 utf8_build_one(char, u_int);
+enum utf8_state	 utf8_from_data(const struct utf8_data *, utf8_char *);
+void		 utf8_to_data(utf8_char, struct utf8_data *);
 void		 utf8_set(struct utf8_data *, u_char);
 void		 utf8_copy(struct utf8_data *, const struct utf8_data *);
 enum utf8_state	 utf8_open(struct utf8_data *, u_char);
diff --git a/usr.bin/tmux/utf8.c b/usr.bin/tmux/utf8.c
index 76e44472384..e91734b3f73 100644
--- a/usr.bin/tmux/utf8.c
+++ b/usr.bin/tmux/utf8.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: utf8.c,v 1.46 2020/05/25 15:02:25 nicm Exp $ */
+/* $OpenBSD: utf8.c,v 1.47 2020/05/25 18:19:29 nicm Exp $ */
 
 /*
  * Copyright (c) 2008 Nicholas Marriott <nicholas.marriott@gmail.com>
@@ -27,153 +27,157 @@
 
 #include "tmux.h"
 
-static int	utf8_width(wchar_t);
-
-struct utf8_big_item {
-	u_int			index;
-	RB_ENTRY(utf8_big_item)	entry;
+struct utf8_item {
+	u_int			offset;
+	RB_ENTRY(utf8_item)	entry;
 
 	char			data[UTF8_SIZE];
 	u_char			size;
 };
-RB_HEAD(utf8_big_tree, utf8_big_item);
+RB_HEAD(utf8_tree, utf8_item);
 
 static int
-utf8_big_cmp(struct utf8_big_item *bi1, struct utf8_big_item *bi2)
+utf8_cmp(struct utf8_item *ui1, struct utf8_item *ui2)
 {
-	if (bi1->size < bi2->size)
+	if (ui1->size < ui2->size)
 		return (-1);
-	if (bi1->size > bi2->size)
+	if (ui1->size > ui2->size)
 		return (1);
-	return (memcmp(bi1->data, bi2->data, bi1->size));
+	return (memcmp(ui1->data, ui2->data, ui1->size));
 }
-RB_GENERATE_STATIC(utf8_big_tree, utf8_big_item, entry, utf8_big_cmp);
-static struct utf8_big_tree utf8_big_tree = RB_INITIALIZER(utf8_big_tree);
+RB_GENERATE_STATIC(utf8_tree, utf8_item, entry, utf8_cmp);
+static struct utf8_tree utf8_tree = RB_INITIALIZER(utf8_tree);
 
-static struct utf8_big_item *utf8_big_list;
-static u_int utf8_big_list_size;
-static u_int utf8_big_list_used;
+static struct utf8_item *utf8_list;
+static u_int		 utf8_list_size;
+static u_int		 utf8_list_used;
 
-union utf8_big_map {
-	u_int	value;
+union utf8_map {
+	utf8_char	uc;
 	struct {
 		u_char	flags;
-#define UTF8_BIG_SIZE 0x1f
-#define UTF8_BIG_WIDTH2 0x20
+#define UTF8_FLAG_SIZE 0x1f
+#define UTF8_FLAG_WIDTH2 0x20
 
 		u_char	data[3];
 	};
 } __packed;
 
-static const union utf8_big_map utf8_big_space1 = {
+static const union utf8_map utf8_space1 = {
 	.flags = 1,
 	.data = " "
 };
-static const union utf8_big_map utf8_big_space2 = {
-	.flags = UTF8_BIG_WIDTH2|2,
+static const union utf8_map utf8_space2 = {
+	.flags = UTF8_FLAG_WIDTH2|2,
 	.data = "  "
 };
 
-/* Get a big item by index. */
-static struct utf8_big_item *
-utf8_get_big_item(const char *data, size_t size)
+/* Get a UTF-8 item by offset. */
+static struct utf8_item *
+utf8_get_item(const char *data, size_t size)
 {
-	struct utf8_big_item bi;
+	struct utf8_item	ui;
+
+	memcpy(ui.data, data, size);
+	ui.size = size;
 
-	memcpy(bi.data, data, size);
-	bi.size = size;
+	return (RB_FIND(utf8_tree, &utf8_tree, &ui));
+}
 
-	return (RB_FIND(utf8_big_tree, &utf8_big_tree, &bi));
+/* Expand UTF-8 list. */
+static int
+utf8_expand_list(void)
+{
+	if (utf8_list_size == 0xffffff)
+		return (-1);
+	if (utf8_list_size == 0)
+		utf8_list_size = 256;
+	else if (utf8_list_size > 0x7fffff)
+		utf8_list_size = 0xffffff;
+	else
+		utf8_list_size *= 2;
+	utf8_list = xreallocarray(utf8_list, utf8_list_size, sizeof *utf8_list);
+	return (0);
 }
 
-/* Add a big item. */
+/* Add a UTF-8 item. */
 static int
-utf8_put_big_item(const char *data, size_t size, u_int *index)
+utf8_put_item(const char *data, size_t size, u_int *offset)
 {
-	struct utf8_big_item	*bi;
+	struct utf8_item	*ui;
 
-	bi = utf8_get_big_item(data, size);
-	if (bi != NULL) {
-		*index = bi->index;
+	ui = utf8_get_item(data, size);
+	if (ui != NULL) {
+		*offset = ui->offset;
 		log_debug("%s: have %.*s at %u", __func__, (int)size, data,
-		    *index);
+		    *offset);
 		return (0);
 	}
 
-	if (utf8_big_list_used == utf8_big_list_size) {
-		if (utf8_big_list_size == 0xffffff)
-			return (-1);
-		if (utf8_big_list_size == 0)
-			utf8_big_list_size = 256;
-		else if (utf8_big_list_size > 0x7fffff)
-			utf8_big_list_size = 0xffffff;
-		else
-			utf8_big_list_size *= 2;
-		utf8_big_list = xreallocarray(utf8_big_list, utf8_big_list_size,
-		    sizeof *utf8_big_list);
-	}
-	*index = utf8_big_list_used++;
+	if (utf8_list_used == utf8_list_size && utf8_expand_list() != 0)
+		return (-1);
+	*offset = utf8_list_used++;
 
-	bi = &utf8_big_list[*index];
-	bi->index = *index;
-	memcpy(bi->data, data, size);
-	bi->size = size;
-	RB_INSERT(utf8_big_tree, &utf8_big_tree, bi);
+	ui = &utf8_list[*offset];
+	ui->offset = *offset;
+	memcpy(ui->data, data, size);
+	ui->size = size;
+	RB_INSERT(utf8_tree, &utf8_tree, ui);
 
-	log_debug("%s: added %.*s at %u", __func__, (int)size, data, *index);
+	log_debug("%s: added %.*s at %u", __func__, (int)size, data, *offset);
 	return (0);
 }
 
-/* Get UTF-8 as index into buffer. */
-u_int
-utf8_map_big(const struct utf8_data *ud)
+/* Get UTF-8 character from data. */
+enum utf8_state
+utf8_from_data(const struct utf8_data *ud, utf8_char *uc)
 {
-	union utf8_big_map	 m = { .value = 0 };
-	u_int			 o;
-	const char		*data = ud->data;
-	size_t			 size = ud->size;
+	union utf8_map	 m = { .uc = 0 };
+	u_int		 offset;
 
 	if (ud->width != 1 && ud->width != 2)
-		return (utf8_big_space1.value);
+		return (utf8_space1.uc);
 
-	if (size > UTF8_BIG_SIZE)
+	if (ud->size > UTF8_FLAG_SIZE)
 		goto fail;
-	if (size == 1)
-		return (utf8_set_big(data[0], 1));
+	if (ud->size == 1)
+		return (utf8_build_one(ud->data[0], 1));
 
-	m.flags = size;
+	m.flags = ud->size;
 	if (ud->width == 2)
-		m.flags |= UTF8_BIG_WIDTH2;
+		m.flags |= UTF8_FLAG_WIDTH2;
 
-	if (size <= 3) {
-		memcpy(&m.data, data, size);
-		return (m.value);
+	if (ud->size <= 3)
+		memcpy(m.data, ud->data, ud->size);
+	else {
+		if (utf8_put_item(ud->data, ud->size, &offset) != 0)
+			goto fail;
+		m.data[0] = (offset & 0xff);
+		m.data[1] = (offset >> 8) & 0xff;
+		m.data[2] = (offset >> 16);
 	}
-
-	if (utf8_put_big_item(data, size, &o) != 0)
-		goto fail;
-	m.data[0] = (o & 0xff);
-	m.data[1] = (o >> 8) & 0xff;
-	m.data[2] = (o >> 16);
-	return (m.value);
+	*uc = m.uc;
+	return (UTF8_DONE);
 
 fail:
 	if (ud->width == 1)
-		return (utf8_big_space1.value);
-	return (utf8_big_space2.value);
+		*uc = utf8_space1.uc;
+	else
+		*uc = utf8_space2.uc;
+	return (UTF8_ERROR);
 }
 
-/* Get UTF-8 from index into buffer. */
+/* Get UTF-8 data from character. */
 void
-utf8_get_big(u_int v, struct utf8_data *ud)
+utf8_to_data(utf8_char uc, struct utf8_data *ud)
 {
-	union utf8_big_map	 m = { .value = v };
-	struct utf8_big_item	*bi;
-	u_int			 o;
+	union utf8_map		 m = { .uc = uc };
+	struct utf8_item	*ui;
+	u_int			 offset;
 
 	memset(ud, 0, sizeof *ud);
-	ud->size = ud->have = (m.flags & UTF8_BIG_SIZE);
-	if (m.flags & UTF8_BIG_WIDTH2)
+	ud->size = ud->have = (m.flags & UTF8_FLAG_SIZE);
+	if (m.flags & UTF8_FLAG_WIDTH2)
 		ud->width = 2;
 	else
 		ud->width = 1;
@@ -183,24 +187,24 @@ utf8_get_big(u_int v, struct utf8_data *ud)
 		return;
 	}
 
-	o = ((u_int)m.data[2] << 16)|((u_int)m.data[1] << 8)|m.data[0];
-	if (o >= utf8_big_list_used)
+	offset = ((u_int)m.data[2] << 16)|((u_int)m.data[1] << 8)|m.data[0];
+	if (offset >= utf8_list_used)
 		memset(ud->data, ' ', ud->size);
 	else {
-		bi = &utf8_big_list[o];
-		memcpy(ud->data, bi->data, ud->size);
+		ui = &utf8_list[offset];
+		memcpy(ud->data, ui->data, ud->size);
 	}
 }
 
-/* Get big value for UTF-8 single character. */
+/* Get UTF-8 character from a single ASCII character. */
 u_int
-utf8_set_big(char c, u_int width)
+utf8_build_one(char c, u_int width)
 {
-	union utf8_big_map	m = { .flags = 1, .data[0] = c };
+	union utf8_map	m = { .flags = 1, .data[0] = c };
 
 	if (width == 2)
-		m.flags |= UTF8_BIG_WIDTH2;
-	return (m.value);
+		m.flags |= UTF8_FLAG_WIDTH2;
+	return (m.uc);
 }
 
 /* Set a single character. */
@@ -225,6 +229,20 @@ utf8_copy(struct utf8_data *to, const struct utf8_data *from)
 		to->data[i] = '\0';
 }
 
+/* Get width of Unicode character. */
+static int
+utf8_width(wchar_t wc)
+{
+	int	width;
+
+	width = wcwidth(wc);
+	if (width < 0 || width > 0xff) {
+		log_debug("Unicode %04lx, wcwidth() %d", (long)wc, width);
+		return (-1);
+	}
+	return (width);
+}
+
 /*
  * Open UTF-8 sequence.
  *
@@ -279,20 +297,6 @@ utf8_append(struct utf8_data *ud, u_char ch)
 	return (UTF8_DONE);
 }
 
-/* Get width of Unicode character. */
-static int
-utf8_width(wchar_t wc)
-{
-	int	width;
-
-	width = wcwidth(wc);
-	if (width < 0 || width > 0xff) {
-		log_debug("Unicode %04lx, wcwidth() %d", (long)wc, width);
-		return (-1);
-	}
-	return (width);
-}
-
 /* Combine UTF-8 into Unicode. */
 enum utf8_state
 utf8_combine(const struct utf8_data *ud, wchar_t *wc)
@@ -337,13 +341,10 @@ int
 utf8_strvis(char *dst, const char *src, size_t len, int flag)
 {
 	struct utf8_data	 ud;
-	const char		*start, *end;
+	const char		*start = dst, *end = src + len;
 	enum utf8_state		 more;
 	size_t			 i;
 
-	start = dst;
-	end = src + len;
-
 	while (src < end) {
 		if ((more = utf8_open(&ud, *src)) == UTF8_MORE) {
 			while (++src < end && more == UTF8_MORE)
@@ -369,7 +370,6 @@ utf8_strvis(char *dst, const char *src, size_t len, int flag)
 			dst = vis(dst, src[0], flag, '\0');
 		src++;
 	}
-
 	*dst = '\0';
 	return (dst - start);
 }
@@ -392,9 +392,9 @@ utf8_stravis(char **dst, const char *src, int flag)
 int
 utf8_isvalid(const char *s)
 {
-	struct utf8_data	 ud;
-	const char		*end;
-	enum utf8_state		 more;
+	struct utf8_data ud;
+	const char	*end;
+	enum utf8_state	 more;
 
 	end = s + strlen(s);
 	while (s < end) {
@@ -420,15 +420,12 @@ utf8_isvalid(const char *s)
 char *
 utf8_sanitize(const char *src)
 {
-	char			*dst;
-	size_t			 n;
-	enum utf8_state		 more;
-	struct utf8_data	 ud;
-	u_int			 i;
+	char		*dst = NULL;
+	size_t		 n = 0;
+	enum utf8_state	 more;
+	struct utf8_data ud;
+	u_int		 i;
 
-	dst = NULL;
-
-	n = 0;
 	while (*src != '\0') {
 		dst = xreallocarray(dst, n + 1, sizeof *dst);
 		if ((more = utf8_open(&ud, *src)) == UTF8_MORE) {
@@ -449,7 +446,6 @@ utf8_sanitize(const char *src)
 			dst[n++] = '_';
 		src++;
 	}
-
 	dst = xreallocarray(dst, n + 1, sizeof *dst);
 	dst[n] = '\0';
 	return (dst);
@@ -471,9 +467,8 @@ u_int
 utf8_strwidth(const struct utf8_data *s, ssize_t n)
 {
 	ssize_t	i;
-	u_int	width;
+	u_int	width = 0;
 
-	width = 0;
 	for (i = 0; s[i].size != 0; i++) {
 		if (n != -1 && n == i)
 			break;
@@ -489,13 +484,10 @@ utf8_strwidth(const struct utf8_data *s, ssize_t n)
 struct utf8_data *
 utf8_fromcstr(const char *src)
 {
-	struct utf8_data	*dst;
-	size_t			 n;
+	struct utf8_data	*dst = NULL;
+	size_t			 n = 0;
 	enum utf8_state		 more;
 
-	dst = NULL;
-
-	n = 0;
 	while (*src != '\0') {
 		dst = xreallocarray(dst, n + 1, sizeof *dst);
 		if ((more = utf8_open(&dst[n], *src)) == UTF8_MORE) {
@@ -511,7 +503,6 @@ utf8_fromcstr(const char *src)
 		n++;
 		src++;
 	}
-
 	dst = xreallocarray(dst, n + 1, sizeof *dst);
 	dst[n].size = 0;
 	return (dst);
@@ -521,18 +512,14 @@ utf8_fromcstr(const char *src)
 char *
 utf8_tocstr(struct utf8_data *src)
 {
-	char	*dst;
-	size_t	 n;
-
-	dst = NULL;
+	char	*dst = NULL;
+	size_t	 n = 0;
 
-	n = 0;
 	for(; src->size != 0; src++) {
 		dst = xreallocarray(dst, n + src->size, 1);
 		memcpy(dst + n, src->data, src->size);
 		n += src->size;
 	}
-
 	dst = xreallocarray(dst, n + 1, 1);
 	dst[n] = '\0';
 	return (dst);
@@ -570,7 +557,7 @@ utf8_padcstr(const char *s, u_int width)
 {
 	size_t	 slen;
 	char	*out;
-	u_int	  n, i;
+	u_int	 n, i;
 
 	n = utf8_cstrwidth(s);
 	if (n >= width)
@@ -591,7 +578,7 @@ utf8_rpadcstr(const char *s, u_int width)
 {
 	size_t	 slen;
 	char	*out;
-	u_int	  n, i;
+	u_int	 n, i;
 
 	n = utf8_cstrwidth(s);
 	if (n >= width)
diff --git a/usr.bin/tmux/window-copy.c b/usr.bin/tmux/window-copy.c
index cc25e6fbc5a..50e037d6bad 100644
--- a/usr.bin/tmux/window-copy.c
+++ b/usr.bin/tmux/window-copy.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: window-copy.c,v 1.290 2020/05/25 12:12:58 nicm Exp $ */
+/* $OpenBSD: window-copy.c,v 1.291 2020/05/25 18:19:29 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicholas.marriott@gmail.com>
@@ -2571,7 +2571,7 @@ window_copy_cellstring(const struct grid_line *gl, u_int px, size_t *size,
 		return (&gce->data.data);
 	}
 
-	utf8_get_big(gl->extddata[gce->offset].data, &ud);
+	utf8_to_data(gl->extddata[gce->offset].data, &ud);
 	*size = ud.size;
 	*allocated = 1;