Re: UTF-8 terminal patch for links-2.1pre12 (was Re: Newlinks version: 2.1pre12)

diff -ub --recursive links-2.1pre12.orig/charsets.c links-2.1pre12/charsets.c
--- links-2.1pre12.orig/charsets.c	2003-02-11 13:54:55.000000000 -0800
+++ links-2.1pre12/charsets.c	2003-09-21 13:23:28.000000000 -0700
@@ -82,7 +82,7 @@
 #define U_EQUAL(a, b) (a).x == (b)
 #define U_ABOVE(a, b) (a).x > (b)
 
-static inline unsigned char *u2cp(int u, int to)
+unsigned char *u2cp(int u, int to)
 {
 	int j, s;
 	if (u < 128) return strings[u];
@@ -142,6 +142,22 @@
 	return utf_buffer;
 }
 
+/* this slow and ugly code is used by the terminal utf_8_io */
+unsigned char *cp2utf_8(int from, int c)
+{
+	int j;
+
+	if (codepages[from].table == table_utf_8) return strings[c];
+	for (j = 0; codepages[from].table[j].c; j++) {
+		if (codepages[from].table[j].c == c)
+		{
+			return encode_utf_8(codepages[from].table[j].u);
+		}
+	}
+	if (c < 128) return strings[c];
+	return encode_utf_8(UCS_NO_CHAR);
+}
+
 void add_utf_8(struct conv_table *ct, int u, unsigned char *str)
 {
 	unsigned char *p = encode_utf_8(u);
diff -ub --recursive links-2.1pre12.orig/default.c links-2.1pre12/default.c
--- links-2.1pre12.orig/default.c	2003-09-01 13:22:54.000000000 -0700
+++ links-2.1pre12/default.c	2003-09-21 13:23:28.000000000 -0700
@@ -634,6 +634,7 @@
 	}
 }
 
+/* terminal NAME(str) MODE(0-3) M11_HACK(0-1) BLOCK_CURSOR.RESTRICT_852.COL(0-7) CHARSET(str) [ UTF_8_IO("utf-8") ]*/
 unsigned char *term_rd(struct option *o, unsigned char *c)
 {
 	struct term_spec *ts;
@@ -644,6 +645,7 @@
 		mem_free(w);
 		goto end;
 	}
+        ts->utf_8_io = 0;
 	mem_free(w);
 	if (!(w = get_token(&c))) goto err;
 	if (strlen(w) != 1 || w[0] < '0' || w[0] > '3') goto err_f;
@@ -663,6 +665,9 @@
 	if ((i = get_cp_index(w)) == -1) goto err_f;
 	ts->charset = i;
 	mem_free(w);
+        if (!(w = get_token(&c))) goto end;
+        if (!(strcasecmp(w, "utf-8"))) ts->utf_8_io = 1;
+        mem_free(w);
 	end:
 	return NULL;
 	err_f:
@@ -671,6 +676,7 @@
 	return "Error reading terminal specification";
 }
 
+/* terminal2 NAME(str) MODE(0-3) M11_HACK(0-1) RESTRICT_852(0-1) COL(0-1) CHARSET(str) [ UTF_8_IO("utf-8") ]*/
 unsigned char *term2_rd(struct option *o, unsigned char *c)
 {
 	struct term_spec *ts;
@@ -681,6 +687,7 @@
 		mem_free(w);
 		goto end;
 	}
+        ts->utf_8_io = 0;
 	mem_free(w);
 	if (!(w = get_token(&c))) goto err;
 	if (strlen(w) != 1 || w[0] < '0' || w[0] > '3') goto err_f;
@@ -702,6 +709,9 @@
 	if ((i = get_cp_index(w)) == -1) goto err_f;
 	ts->charset = i;
 	mem_free(w);
+        if (!(w = get_token(&c))) goto end;
+        if (!(strcasecmp(w, "utf-8"))) ts->utf_8_io = 1;
+        mem_free(w);
 	end:
 	return NULL;
 	err_f:
@@ -724,6 +734,10 @@
 		add_num_to_str(s, l, !!ts->col + !!ts->restrict_852 * 2 + !!ts->block_cursor * 4);
 		add_to_str(s, l, " ");
 		add_to_str(s, l, get_cp_mime_name(ts->charset));
+                if (ts->utf_8_io)
+                {
+                    add_to_str(s, l, " utf-8");
+                }
 	}
 }
 
diff -ub --recursive links-2.1pre12.orig/intl/english.lng links-2.1pre12/intl/english.lng
--- links-2.1pre12.orig/intl/english.lng	2003-09-01 13:23:00.000000000 -0700
+++ links-2.1pre12/intl/english.lng	2003-09-21 13:23:39.000000000 -0700
@@ -53,6 +53,7 @@
 T_LINUX_OR_OS2_FRAMES, "Linux or OS/2 frames",
 T_KOI8R_FRAMES, "KOI8-R frames",
 T_USE_11M, "Use ^[[11m",
+T_UTF_8_IO, "UTF-8 I/O",
 T_RESTRICT_FRAMES_IN_CP850_852, "Restrict frames in cp850/852",
 T_BLOCK_CURSOR, "Block cursor",
 T_COLOR, "Color",
diff -ub --recursive links-2.1pre12.orig/links.h links-2.1pre12/links.h
--- links-2.1pre12.orig/links.h	2003-09-18 04:46:34.000000000 -0700
+++ links-2.1pre12/links.h	2003-09-21 13:25:58.000000000 -0700
@@ -1708,6 +1708,11 @@
 #ifdef G
 	struct graphics_device *dev;
 #endif
+	struct {
+		int ucs;
+		int len;
+		int min;
+	} utf_8;
 };
 
 struct term_spec {
@@ -1716,6 +1721,7 @@
 	unsigned char term[MAX_TERM_LEN];
 	int mode;
 	int m11_hack;
+	int utf_8_io;
 	int restrict_852;
 	int block_cursor;
 	int col;
@@ -3163,6 +3169,9 @@
 
 /* charsets.c */
 
+/* UCS/Unicode replacement character */
+#define UCS_NO_CHAR 0xFFFD
+
 #include "codepage.h"
 
 struct conv_table {
@@ -3183,6 +3192,8 @@
 void free_conv_table(void);
 unsigned char *encode_utf_8(int);
 int cp2u(unsigned char, int);
+unsigned char *cp2utf_8(int, int);
+unsigned char *u2cp(int, int);
 
 #ifdef G
 int get_utf_8(unsigned char **p);
diff -ub --recursive links-2.1pre12.orig/menu.c links-2.1pre12/menu.c
--- links-2.1pre12.orig/menu.c	2003-09-01 13:22:57.000000000 -0700
+++ links-2.1pre12/menu.c	2003-09-21 13:23:39.000000000 -0700
@@ -462,15 +462,15 @@
 	cls_redraw_all_terminals();
 }
 
-unsigned char *td_labels[] = { TEXT(T_NO_FRAMES), TEXT(T_VT_100_FRAMES), TEXT(T_LINUX_OR_OS2_FRAMES), TEXT(T_KOI8R_FRAMES), TEXT(T_USE_11M), TEXT(T_RESTRICT_FRAMES_IN_CP850_852), TEXT(T_BLOCK_CURSOR), TEXT(T_COLOR), NULL };
+unsigned char *td_labels[] = { TEXT(T_NO_FRAMES), TEXT(T_VT_100_FRAMES), TEXT(T_LINUX_OR_OS2_FRAMES), TEXT(T_KOI8R_FRAMES), TEXT(T_USE_11M), TEXT(T_RESTRICT_FRAMES_IN_CP850_852), TEXT(T_BLOCK_CURSOR), TEXT(T_COLOR), TEXT(T_UTF_8_IO), NULL };
 
 void terminal_options(struct terminal *term, void *xxx, struct session *ses)
 {
 	struct dialog *d;
 	struct term_spec *ts = new_term_spec(term->term);
 	if (!ts) return;
-	if (!(d = mem_alloc(sizeof(struct dialog) + 11 * sizeof(struct dialog_item)))) return;
-	memset(d, 0, sizeof(struct dialog) + 11 * sizeof(struct dialog_item));
+	if (!(d = mem_alloc(sizeof(struct dialog) + 12 * sizeof(struct dialog_item)))) return;
+	memset(d, 0, sizeof(struct dialog) + 12 * sizeof(struct dialog_item));
 	d->title = TEXT(T_TERMINAL_OPTIONS);
 	d->fn = checkbox_list_fn;
 	d->udata = td_labels;
@@ -511,15 +511,19 @@
 	d->items[7].gid = 0;
 	d->items[7].dlen = sizeof(int);
 	d->items[7].data = (void *)&ts->col;
-	d->items[8].type = D_BUTTON;
-	d->items[8].gid = B_ENTER;
-	d->items[8].fn = ok_dialog;
-	d->items[8].text = TEXT(T_OK);
+	d->items[8].type = D_CHECKBOX;
+	d->items[8].gid = 0;
+	d->items[8].dlen = sizeof(int);
+	d->items[8].data = (void *)&ts->utf_8_io;
 	d->items[9].type = D_BUTTON;
-	d->items[9].gid = B_ESC;
-	d->items[9].fn = cancel_dialog;
-	d->items[9].text = TEXT(T_CANCEL);
-	d->items[10].type = D_END;
+	d->items[9].gid = B_ENTER;
+	d->items[9].fn = ok_dialog;
+	d->items[9].text = TEXT(T_OK);
+	d->items[10].type = D_BUTTON;
+	d->items[10].gid = B_ESC;
+	d->items[10].fn = cancel_dialog;
+	d->items[10].text = TEXT(T_CANCEL);
+	d->items[11].type = D_END;
  	do_dialog(term, d, getml(d, NULL));
 }
 
diff -ub --recursive links-2.1pre12.orig/terminal.c links-2.1pre12/terminal.c
--- links-2.1pre12.orig/terminal.c	2003-09-01 13:22:57.000000000 -0700
+++ links-2.1pre12/terminal.c	2003-09-21 13:23:39.000000000 -0700
@@ -694,6 +694,25 @@
 
 #endif
 
+static inline void term_send_event(struct terminal *term, struct event *ev)
+{
+	((struct window *)&term->windows)->next->handler(term->windows.next, ev, 0);
+}
+
+static inline void term_send_ucs(struct terminal *term, struct event *ev, int u)
+{
+	unsigned char *recoded;
+
+	if (u == 0xA0) u = ' ';
+	recoded = u2cp(u, term->spec->charset);
+	if (! recoded) recoded = "*";
+	while (*recoded) {
+		ev->x = *recoded;
+		term_send_event(term, ev);
+		recoded ++;
+	}
+}
+
 void in_term(struct terminal *term)
 {
 	struct event *ev;
@@ -755,7 +774,40 @@
 			goto send_redraw;
 		}
 		else if (ev->ev == EV_KBD && ev->x == KBD_CTRL_C) ((struct window *)&term->windows)->prev->handler(term->windows.prev, ev, 0);
-		else ((struct window *)&term->windows)->next->handler(term->windows.next, ev, 0);
+		else if (ev->ev == EV_KBD) {
+			if (term->utf_8.len) {
+				if ((ev->x & 0xC0) == 0x80 && term->spec->utf_8_io) {
+					term->utf_8.ucs <<= 6;
+					term->utf_8.ucs |= ev->x & 0x3F;
+					if (! --term->utf_8.len) {
+						int u = term->utf_8.ucs;
+
+						if (u < term->utf_8.min) u = UCS_NO_CHAR;
+						term_send_ucs(term, ev, u);
+					}
+					goto mm;
+				} else {
+					term->utf_8.len = 0;
+					term_send_ucs(term, ev, UCS_NO_CHAR);
+				}
+			}
+			if (ev->x < 0x80 || ev->x > 0xFF || ! term->spec->utf_8_io) {
+				term_send_event(term, ev);
+				goto mm;
+			} else if ((ev->x & 0xC0) == 0xC0 && (ev->x & 0xFE) != 0xFE) {
+				int mask, len = 0, cov = 0x80;
+
+				for (mask = 0x80; ev->x & mask; mask >>= 1) {
+					len++;
+					term->utf_8.min = cov;
+					cov = 1 << (1 + 5 * len);
+				}
+				term->utf_8.len = len - 1;
+				term->utf_8.ucs = ev->x & (mask - 1);
+				goto mm;
+			}
+			term_send_ucs(term, ev, UCS_NO_CHAR);
+		} else term_send_event(term, ev);
 	}
 	if (ev->ev == EV_ABORT) destroy_terminal(term);
 	/*redraw_screen(term);*/
@@ -772,6 +824,14 @@
 
 unsigned char frame_dumb[48] =	"   ||||++||++++++--|-+||++--|-+----++++++++     ";
 unsigned char frame_vt100[48] =	"aaaxuuukkuxkjjjkmvwtqnttmlvwtqnvvwwmmllnnjla    ";
+unsigned char frame_vt100_u[48] = {
+	177,177,177,179,180,180,180,191,
+	191,180,179,191,217,217,217,191,
+	192,193,194,195,196,197,195,195,
+	192,218,193,194,195,196,197,193,
+	193,194,194,192,192,218,218,197,
+	197,217,218,177, 32, 32, 32, 32
+};
 unsigned char frame_koi[48] = {
 	144,145,146,129,135,178,180,167,
 	166,181,161,168,174,173,172,131,
@@ -795,7 +855,7 @@
 	unsigned char c = ch & 0xff;							\
 	unsigned char A = ch >> 8 & 0x7f;						\
 	if (s->mode == TERM_LINUX) {							\
-		if (s->m11_hack) {							\
+		if (s->m11_hack && !s->utf_8_io) {					\
 			if (ch >> 15 != mode) {						\
 				if (!(mode = ch >> 15)) add_to_str(&a, &l, "\033[10m");	\
 				else add_to_str(&a, &l, "\033[11m");			\
@@ -804,12 +864,14 @@
 		if (s->restrict_852 && (ch >> 15) && c >= 176 && c < 224) {		\
 			if (frame_restrict[c - 176]) c = frame_restrict[c - 176];	\
 		}									\
-	} else if (s->mode == TERM_VT100) {						\
+	} else if (s->mode == TERM_VT100 && !s->utf_8_io) {				\
 		if (ch >> 15 != mode) {							\
 			if (!(mode = ch >> 15)) add_to_str(&a, &l, "\x0f");		\
 			else add_to_str(&a, &l, "\x0e");				\
 		}									\
 		if (mode && c >= 176 && c < 224) c = frame_vt100[c - 176];		\
+	} else if (s->mode == TERM_VT100 && (ch >> 15) && c >= 176 && c < 224) {	\
+		c = frame_vt100_u[c - 176];						\
 	} else if (s->mode == TERM_KOI8 && (ch >> 15) && c >= 176 && c < 224) { c = frame_koi[c - 176];\
 	} else if (s->mode == TERM_DUMB && (ch >> 15) && c >= 176 && c < 224) c = frame_dumb[c - 176];\
 	if (!(A & 0100) && (A >> 3) == (A & 7)) A = (A & 070) | 7 * !(A & 020);		\
@@ -829,7 +891,20 @@
 		if (attrib & 0100) add_to_str(&a, &l, ";1");				\
 		add_to_str(&a, &l, "m");						\
 	}										\
-	if (c >= ' ' && c != 127/* && c != 155*/) add_chr_to_str(&a, &l, c);		\
+	if (c >= ' ' && c != 127/* && c != 155*/) {					\
+		int charset = s->charset;						\
+											\
+		if (ch >> 15) {								\
+			int frames_charset = (s->mode == TERM_LINUX || s->mode == TERM_VT100) \
+						? get_cp_index("cp437")			\
+						: s->mode == TERM_KOI8			\
+						? get_cp_index("koi8-r")		\
+						: -1;					\
+			if (frames_charset != -1) charset = frames_charset;		\
+		}									\
+		if (s->utf_8_io) add_to_str(&a, &l, cp2utf_8(charset, c));		\
+		else add_chr_to_str(&a, &l, c);						\
+	}										\
 	else if (!c || c == 1) add_chr_to_str(&a, &l, ' ');				\
 	else add_chr_to_str(&a, &l, '.');						\
 	cx++;										\