viewer: Improve display of messages with Unicode.

* Convert Unicode subjects to ASCII-approximation. To do this,
  we use the anyascii library, which seems to do a good job.
  This is better than displaying a ? if we can't display the
  character.
* Stop displaying messages on NUL rather than aborting.

anyascii.h

@@ -0,0 +1,34 @@
+/*
+ISC License
+
+Copyright (c) 2020-2023, Hunter WB <hunterwb.com>
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+
+#ifndef ANYASCII_H
+#define ANYASCII_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+/**
+ * Gets the ASCII transliteration of a Unicode code point
+ * 
+ * @param utf32 A Unicode code point
+ * @param ascii A pointer for the result to be written to; not null-terminated
+ * @return The number of chars in *ascii
+ */
+size_t anyascii(uint_least32_t utf32, const char **ascii);
+
+#endif

menu.c

@@ -180,6 +180,34 @@ void cleanup_folder_menu(struct client *client)
 	}
 }
 
+#include "anyascii.h"
+#include "utf8.h"
+
+static char ascii[4096];
+
+static void anyascii_string(const char *in, char *out)
+{
+	uint32_t utf32;
+	uint32_t state = 0;
+	const char *r;
+	size_t rlen;
+
+	for (; *in; in++) {
+		utf8_decode(&state, &utf32, (unsigned char) *in);
+		switch (state) {
+		case UTF8_ACCEPT:;
+			rlen = anyascii(utf32, &r);
+			memcpy(out, r, rlen);
+			out += rlen;
+			break;
+		case UTF8_REJECT:
+			state = UTF8_ACCEPT;
+			break;
+		}
+	}
+	*out = 0;
+}
+
 static inline void format_subject(char *restrict buf, size_t len, int width, const char *s)
 {
 	if (!s) {
@@ -189,7 +217,8 @@ static inline void format_subject(char *restrict buf, size_t len, int width, con
 	if (!strncasecmp(s, "Re: ", 4)) {
 		s += 4;
 	}
-	safe_strncpy(buf, s, len);
+	anyascii_string(s, ascii);
+	safe_strncpy(buf, ascii, len);
 	if (strlen(buf) > (size_t) width) {
 		buf[width - 1] = '.';
 		buf[width - 2] = '.';

messages.c

@@ -152,6 +152,9 @@ void delete_message(struct client *client, struct message *msg)
 	free_message(client, msg);
 }
 
+/* XXX For messages like move, when called from viewer (as opposed to message list),
+ * the number of messages is changed after we return, which clears the status pane,
+ * so the status update is essentially lost. */
 int handle_message_op(struct client *client, struct pollfd *pfds, struct message *msg, struct message_data *mdata, int c)
 {
 	int res, cur;
@@ -326,7 +329,7 @@ int handle_message_op(struct client *client, struct pollfd *pfds, struct message
 	}
 
 	/* Success, but redraw since stuff changed.
-	 * In all cases, both the message and folder panes have chaned. */
+	 * In all cases, both the message and folder panes have changed. */
 	return 2;
 }
 

utf8.c

@@ -0,0 +1,55 @@
+/*
+Copyright (c) 2008-2009 Bjoern Hoehrmann <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to
+do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#include <stdint.h>
+#include "utf8.h"
+
+static const uint8_t utf8d[] = {
+  // The first part of the table maps bytes to character classes that
+  // to reduce the size of the transition table and create bitmasks.
+   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+   0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+   1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,  9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,
+   7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,  7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,
+   8,8,2,2,2,2,2,2,2,2,2,2,2,2,2,2,  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
+  10,3,3,3,3,3,3,3,3,3,3,3,3,4,3,3, 11,6,6,6,5,8,8,8,8,8,8,8,8,8,8,8,
+
+  // The second part is a transition table that maps a combination
+  // of a state of the automaton and a character class to a state.
+   0,12,24,36,60,96,84,12,12,12,48,72, 12,12,12,12,12,12,12,12,12,12,12,12,
+  12, 0,12,12,12,12,12, 0,12, 0,12,12, 12,24,12,12,12,12,12,24,12,24,12,12,
+  12,12,12,12,12,12,12,24,12,12,12,12, 12,24,12,12,12,12,12,12,12,24,12,12,
+  12,12,12,12,12,12,12,36,12,36,12,12, 12,36,12,12,12,12,12,36,12,36,12,12,
+  12,36,12,12,12,12,12,12,12,12,12,12
+};
+
+void utf8_decode(uint32_t *state, uint32_t *utf32, uint32_t byte) {
+  uint32_t type = utf8d[byte];
+
+  *utf32 = (*state != UTF8_ACCEPT) ?
+    (byte & 0x3fu) | (*utf32 << 6) :
+    (0xffu >> type) & (byte);
+
+  *state = utf8d[256 + *state + type];
+}

utf8.h

@@ -0,0 +1,11 @@
+#include <stdint.h>
+
+#ifndef UTF8_H
+#define UTF8_H
+
+#define UTF8_ACCEPT 0
+#define UTF8_REJECT 12
+
+void utf8_decode(uint32_t *state, uint32_t *utf32, uint32_t byte);
+
+#endif

viewer.c

@@ -197,6 +197,12 @@ static int pad_add(WINDOW *pad, const char *restrict data, size_t len, int flags
 	/* We can't output carriage returns to the screen */
 	for (; i < len; i++, c++, outputcol++, col++) {
 		getyx(pad, y, x);
+		if (*c == 0) {
+			/* NUL character???
+			 * Shouldn't happen, but if it does, message is over */
+			client_debug(1, "WARNING: Parsed NUL character in message output?");
+			break;
+		}
 		if (x != outputcol) {
 			client_debug(1, "WARNING! outputcol is %d, but actually at row %d, col %d? (char %d: '%c')", outputcol, y, x, *c, isprint(*c) ? *c : ' ');
 			if (x <= 127) {